fix: set default batch size (#597)

renovate/github.com-imdario-mergo-1.x
Ettore Di Giacinto 1 year ago committed by GitHub
parent 1b7990d5d9
commit 10ddd72b58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      api/prediction.go

@ -44,6 +44,8 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
llamaOpts = append(llamaOpts, llama.SetTensorSplit(c.TensorSplit)) llamaOpts = append(llamaOpts, llama.SetTensorSplit(c.TensorSplit))
if c.Batch != 0 { if c.Batch != 0 {
llamaOpts = append(llamaOpts, llama.SetNBatch(c.Batch)) llamaOpts = append(llamaOpts, llama.SetNBatch(c.Batch))
} else {
llamaOpts = append(llamaOpts, llama.SetNBatch(512))
} }
return llamaOpts return llamaOpts

Loading…
Cancel
Save