fix: set default batch size (#597)

1 year ago · 10ddd72b58
parent 1b7990d5d9
commit 10ddd72b58
1 changed files with 2 additions and 0 deletions
--- a/api/prediction.go
+++ b/api/prediction.go
@ -44,6 +44,8 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
 	llamaOpts = append(llamaOpts, llama.SetTensorSplit(c.TensorSplit))
 	if c.Batch != 0 {
 		llamaOpts = append(llamaOpts, llama.SetNBatch(c.Batch))
+	} else {
+		llamaOpts = append(llamaOpts, llama.SetNBatch(512))
 	}

 	return llamaOpts