From 3593cb0c8797cfeb5f2c74744c28ee81d5ed9d94 Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@users.noreply.github.com>
Date: Tue, 27 Jun 2023 09:00:10 +0200
Subject: [PATCH] feat: update llama, enable NUMA (#684)

---
 Makefile          | 2 +-
 api/config.go     | 1 +
 api/prediction.go | 4 ++++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 4b8b149..9a65a6e 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ GOTEST=$(GOCMD) test
 GOVET=$(GOCMD) vet
 BINARY_NAME=local-ai
 
-GOLLAMA_VERSION?=9470597ae75ad8b5f17cfab73805ee4a6685d199
+GOLLAMA_VERSION?=f104111358e8098aea69ce408b85b707528179ef
 GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all
 GPT4ALL_VERSION?=3417a37c5472fb5111a7bd0ed747c8df749c595e
 GOGGMLTRANSFORMERS_VERSION?=a459d2726792132541152c981ed9fbfe28f4fd20
diff --git a/api/config.go b/api/config.go
index 67aec35..ba84e0d 100644
--- a/api/config.go
+++ b/api/config.go
@@ -23,6 +23,7 @@ type Config struct {
 	TrimSpace      []string          `yaml:"trimspace"`
 	ContextSize    int               `yaml:"context_size"`
 	F16            bool              `yaml:"f16"`
+	NUMA           bool              `yaml:"numa"`
 	Threads        int               `yaml:"threads"`
 	Debug          bool              `yaml:"debug"`
 	Roles          map[string]string `yaml:"roles"`
diff --git a/api/prediction.go b/api/prediction.go
index 87eea34..bc23d86 100644
--- a/api/prediction.go
+++ b/api/prediction.go
@@ -48,6 +48,10 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
 		llamaOpts = append(llamaOpts, llama.SetNBatch(512))
 	}
 
+	if c.NUMA {
+		llamaOpts = append(llamaOpts, llama.EnableNUMA)
+	}
+
 	if c.LowVRAM {
 		llamaOpts = append(llamaOpts, llama.EnabelLowVRAM)
 	}