feat: add typical_p to model parameters (#598)

Signed-off-by: mudler <mudler@mocaccino.org>

api/config.go

@@ -236,6 +236,10 @@ func updateConfig(config *Config, input *OpenAIRequest) {
 		config.MirostatTAU = input.MirostatTAU
 	}
 
+	if input.TypicalP != 0 {
+		config.TypicalP = input.TypicalP
+	}
+
 	switch inputs := input.Input.(type) {
 	case string:
 		if inputs != "" {

api/openai.go

@@ -133,6 +133,8 @@ type OpenAIRequest struct {
 	// Image (not supported by OpenAI)
 	Mode int `json:"mode"`
 	Step int `json:"step"`
+
+	TypicalP float64 `json:"typical_p" yaml:"typical_p"`
 }
 
 func defaultRequest(modelFile string) OpenAIRequest {

api/prediction.go

@@ -238,6 +238,7 @@ func buildLLamaPredictOptions(c Config, modelPath string) []llama.PredictOption
 	predictOptions = append(predictOptions, llama.SetPredictionMainGPU(c.MainGPU))
 	predictOptions = append(predictOptions, llama.SetPredictionTensorSplit(c.TensorSplit))
 	predictOptions = append(predictOptions, llama.SetTailFreeSamplingZ(c.TFZ))
+	predictOptions = append(predictOptions, llama.SetTypicalP(c.TypicalP))
 
 	return predictOptions
 }