From 10ddd72b58e54ca847733fe305ddea9451e49732 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 14 Jun 2023 19:09:27 +0200 Subject: [PATCH] fix: set default batch size (#597) --- api/prediction.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/prediction.go b/api/prediction.go index ff4866f..7a49a57 100644 --- a/api/prediction.go +++ b/api/prediction.go @@ -44,6 +44,8 @@ func defaultLLamaOpts(c Config) []llama.ModelOption { llamaOpts = append(llamaOpts, llama.SetTensorSplit(c.TensorSplit)) if c.Batch != 0 { llamaOpts = append(llamaOpts, llama.SetNBatch(c.Batch)) + } else { + llamaOpts = append(llamaOpts, llama.SetNBatch(512)) } return llamaOpts