This finally makes everything more consistent Signed-off-by: Ettore Di Giacinto <mudler@localai.io>renovate/github.com-imdario-mergo-1.x
Ettore Di Giacinto
1 year ago
parent
5dcfdbe51d
commit
1d0ed95a54
@ -0,0 +1,22 @@ |
||||
package main |
||||
|
||||
// Note: this is started internally by LocalAI and a server is allocated for each model
|
||||
|
||||
import ( |
||||
"flag" |
||||
|
||||
grpc "github.com/go-skynet/LocalAI/pkg/grpc" |
||||
bert "github.com/go-skynet/LocalAI/pkg/grpc/llm/bert" |
||||
) |
||||
|
||||
var ( |
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to") |
||||
) |
||||
|
||||
func main() { |
||||
flag.Parse() |
||||
|
||||
if err := grpc.StartServer(*addr, &bert.Embeddings{}); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
package main |
||||
|
||||
// Note: this is started internally by LocalAI and a server is allocated for each model
|
||||
|
||||
import ( |
||||
"flag" |
||||
|
||||
bloomz "github.com/go-skynet/LocalAI/pkg/grpc/llm/bloomz" |
||||
|
||||
grpc "github.com/go-skynet/LocalAI/pkg/grpc" |
||||
) |
||||
|
||||
var ( |
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to") |
||||
) |
||||
|
||||
func main() { |
||||
flag.Parse() |
||||
|
||||
if err := grpc.StartServer(*addr, &bloomz.LLM{}); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
package main |
||||
|
||||
// Note: this is started internally by LocalAI and a server is allocated for each model
|
||||
|
||||
import ( |
||||
"flag" |
||||
|
||||
transformers "github.com/go-skynet/LocalAI/pkg/grpc/llm/transformers" |
||||
|
||||
grpc "github.com/go-skynet/LocalAI/pkg/grpc" |
||||
) |
||||
|
||||
var ( |
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to") |
||||
) |
||||
|
||||
func main() { |
||||
flag.Parse() |
||||
|
||||
if err := grpc.StartServer(*addr, &transformers.Falcon{}); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
package main |
||||
|
||||
// Note: this is started internally by LocalAI and a server is allocated for each model
|
||||
|
||||
import ( |
||||
"flag" |
||||
|
||||
langchain "github.com/go-skynet/LocalAI/pkg/grpc/llm/langchain" |
||||
|
||||
grpc "github.com/go-skynet/LocalAI/pkg/grpc" |
||||
) |
||||
|
||||
var ( |
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to") |
||||
) |
||||
|
||||
func main() { |
||||
flag.Parse() |
||||
|
||||
if err := grpc.StartServer(*addr, &langchain.LLM{}); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
package main |
||||
|
||||
// Note: this is started internally by LocalAI and a server is allocated for each model
|
||||
|
||||
import ( |
||||
"flag" |
||||
|
||||
tts "github.com/go-skynet/LocalAI/pkg/grpc/tts" |
||||
|
||||
grpc "github.com/go-skynet/LocalAI/pkg/grpc" |
||||
) |
||||
|
||||
var ( |
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to") |
||||
) |
||||
|
||||
func main() { |
||||
flag.Parse() |
||||
|
||||
if err := grpc.StartServer(*addr, &tts.Piper{}); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
package main |
||||
|
||||
// Note: this is started internally by LocalAI and a server is allocated for each model
|
||||
|
||||
import ( |
||||
"flag" |
||||
|
||||
rwkv "github.com/go-skynet/LocalAI/pkg/grpc/llm/rwkv" |
||||
|
||||
grpc "github.com/go-skynet/LocalAI/pkg/grpc" |
||||
) |
||||
|
||||
var ( |
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to") |
||||
) |
||||
|
||||
func main() { |
||||
flag.Parse() |
||||
|
||||
if err := grpc.StartServer(*addr, &rwkv.LLM{}); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
package main |
||||
|
||||
// Note: this is started internally by LocalAI and a server is allocated for each model
|
||||
|
||||
import ( |
||||
"flag" |
||||
|
||||
image "github.com/go-skynet/LocalAI/pkg/grpc/image" |
||||
|
||||
grpc "github.com/go-skynet/LocalAI/pkg/grpc" |
||||
) |
||||
|
||||
var ( |
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to") |
||||
) |
||||
|
||||
func main() { |
||||
flag.Parse() |
||||
|
||||
if err := grpc.StartServer(*addr, &image.StableDiffusion{}); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
package main |
||||
|
||||
// Note: this is started internally by LocalAI and a server is allocated for each model
|
||||
|
||||
import ( |
||||
"flag" |
||||
|
||||
transcribe "github.com/go-skynet/LocalAI/pkg/grpc/transcribe" |
||||
|
||||
grpc "github.com/go-skynet/LocalAI/pkg/grpc" |
||||
) |
||||
|
||||
var ( |
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to") |
||||
) |
||||
|
||||
func main() { |
||||
flag.Parse() |
||||
|
||||
if err := grpc.StartServer(*addr, &transcribe.Whisper{}); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,42 @@ |
||||
package base |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
"fmt" |
||||
|
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
"github.com/go-skynet/LocalAI/pkg/grpc/whisper/api" |
||||
) |
||||
|
||||
type Base struct { |
||||
} |
||||
|
||||
func (llm *Base) Load(opts *pb.ModelOptions) error { |
||||
return fmt.Errorf("unimplemented") |
||||
|
||||
} |
||||
|
||||
func (llm *Base) Predict(opts *pb.PredictOptions) (string, error) { |
||||
return "", fmt.Errorf("unimplemented") |
||||
} |
||||
|
||||
func (llm *Base) PredictStream(opts *pb.PredictOptions, results chan string) error { |
||||
return fmt.Errorf("unimplemented") |
||||
} |
||||
|
||||
func (llm *Base) Embeddings(opts *pb.PredictOptions) ([]float32, error) { |
||||
return []float32{}, fmt.Errorf("unimplemented") |
||||
} |
||||
|
||||
func (llm *Base) GenerateImage(*pb.GenerateImageRequest) error { |
||||
return fmt.Errorf("unimplemented") |
||||
} |
||||
|
||||
func (llm *Base) AudioTranscription(*pb.TranscriptRequest) (api.Result, error) { |
||||
return api.Result{}, fmt.Errorf("unimplemented") |
||||
} |
||||
|
||||
func (llm *Base) TTS(*pb.TTSRequest) error { |
||||
return fmt.Errorf("unimplemented") |
||||
} |
||||
@ -0,0 +1,33 @@ |
||||
package image |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
"github.com/go-skynet/LocalAI/pkg/grpc/base" |
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
"github.com/go-skynet/LocalAI/pkg/stablediffusion" |
||||
) |
||||
|
||||
type StableDiffusion struct { |
||||
base.Base |
||||
stablediffusion *stablediffusion.StableDiffusion |
||||
} |
||||
|
||||
func (sd *StableDiffusion) Load(opts *pb.ModelOptions) error { |
||||
var err error |
||||
// Note: the Model here is a path to a directory containing the model files
|
||||
sd.stablediffusion, err = stablediffusion.New(opts.Model) |
||||
return err |
||||
} |
||||
|
||||
func (sd *StableDiffusion) GenerateImage(opts *pb.GenerateImageRequest) error { |
||||
return sd.stablediffusion.GenerateImage( |
||||
int(opts.Height), |
||||
int(opts.Width), |
||||
int(opts.Mode), |
||||
int(opts.Step), |
||||
int(opts.Seed), |
||||
opts.PositivePrompt, |
||||
opts.NegativePrompt, |
||||
opts.Dst) |
||||
} |
||||
@ -0,0 +1,33 @@ |
||||
package bert |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
bert "github.com/go-skynet/go-bert.cpp" |
||||
|
||||
"github.com/go-skynet/LocalAI/pkg/grpc/base" |
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
) |
||||
|
||||
type Embeddings struct { |
||||
base.Base |
||||
bert *bert.Bert |
||||
} |
||||
|
||||
func (llm *Embeddings) Load(opts *pb.ModelOptions) error { |
||||
model, err := bert.New(opts.Model) |
||||
llm.bert = model |
||||
return err |
||||
} |
||||
|
||||
func (llm *Embeddings) Embeddings(opts *pb.PredictOptions) ([]float32, error) { |
||||
if len(opts.EmbeddingTokens) > 0 { |
||||
tokens := []int{} |
||||
for _, t := range opts.EmbeddingTokens { |
||||
tokens = append(tokens, int(t)) |
||||
} |
||||
return llm.bert.TokenEmbeddings(tokens, bert.SetThreads(int(opts.Threads))) |
||||
} |
||||
|
||||
return llm.bert.Embeddings(opts.Embeddings, bert.SetThreads(int(opts.Threads))) |
||||
} |
||||
@ -0,0 +1,59 @@ |
||||
package bloomz |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/go-skynet/LocalAI/pkg/grpc/base" |
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
|
||||
"github.com/go-skynet/bloomz.cpp" |
||||
) |
||||
|
||||
type LLM struct { |
||||
base.Base |
||||
|
||||
bloomz *bloomz.Bloomz |
||||
} |
||||
|
||||
func (llm *LLM) Load(opts *pb.ModelOptions) error { |
||||
model, err := bloomz.New(opts.Model) |
||||
llm.bloomz = model |
||||
return err |
||||
} |
||||
|
||||
func buildPredictOptions(opts *pb.PredictOptions) []bloomz.PredictOption { |
||||
predictOptions := []bloomz.PredictOption{ |
||||
bloomz.SetTemperature(float64(opts.Temperature)), |
||||
bloomz.SetTopP(float64(opts.TopP)), |
||||
bloomz.SetTopK(int(opts.TopK)), |
||||
bloomz.SetTokens(int(opts.Tokens)), |
||||
bloomz.SetThreads(int(opts.Threads)), |
||||
} |
||||
|
||||
if opts.Seed != 0 { |
||||
predictOptions = append(predictOptions, bloomz.SetSeed(int(opts.Seed))) |
||||
} |
||||
|
||||
return predictOptions |
||||
} |
||||
|
||||
func (llm *LLM) Predict(opts *pb.PredictOptions) (string, error) { |
||||
return llm.bloomz.Predict(opts.Prompt, buildPredictOptions(opts)...) |
||||
} |
||||
|
||||
// fallback to Predict
|
||||
func (llm *LLM) PredictStream(opts *pb.PredictOptions, results chan string) error { |
||||
go func() { |
||||
res, err := llm.bloomz.Predict(opts.Prompt, buildPredictOptions(opts)...) |
||||
|
||||
if err != nil { |
||||
fmt.Println("err: ", err) |
||||
} |
||||
results <- res |
||||
close(results) |
||||
}() |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,58 @@ |
||||
package langchain |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/go-skynet/LocalAI/pkg/grpc/base" |
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
"github.com/go-skynet/LocalAI/pkg/langchain" |
||||
) |
||||
|
||||
type LLM struct { |
||||
base.Base |
||||
|
||||
langchain *langchain.HuggingFace |
||||
model string |
||||
} |
||||
|
||||
func (llm *LLM) Load(opts *pb.ModelOptions) error { |
||||
llm.langchain, _ = langchain.NewHuggingFace(opts.Model) |
||||
llm.model = opts.Model |
||||
return nil |
||||
} |
||||
|
||||
func (llm *LLM) Predict(opts *pb.PredictOptions) (string, error) { |
||||
o := []langchain.PredictOption{ |
||||
langchain.SetModel(llm.model), |
||||
langchain.SetMaxTokens(int(opts.Tokens)), |
||||
langchain.SetTemperature(float64(opts.Temperature)), |
||||
langchain.SetStopWords(opts.StopPrompts), |
||||
} |
||||
pred, err := llm.langchain.PredictHuggingFace(opts.Prompt, o...) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
return pred.Completion, nil |
||||
} |
||||
|
||||
func (llm *LLM) PredictStream(opts *pb.PredictOptions, results chan string) error { |
||||
o := []langchain.PredictOption{ |
||||
langchain.SetModel(llm.model), |
||||
langchain.SetMaxTokens(int(opts.Tokens)), |
||||
langchain.SetTemperature(float64(opts.Temperature)), |
||||
langchain.SetStopWords(opts.StopPrompts), |
||||
} |
||||
go func() { |
||||
res, err := llm.langchain.PredictHuggingFace(opts.Prompt, o...) |
||||
|
||||
if err != nil { |
||||
fmt.Println("err: ", err) |
||||
} |
||||
results <- res.Completion |
||||
close(results) |
||||
}() |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,71 @@ |
||||
package rwkv |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
"fmt" |
||||
"path/filepath" |
||||
|
||||
"github.com/donomii/go-rwkv.cpp" |
||||
"github.com/go-skynet/LocalAI/pkg/grpc/base" |
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
) |
||||
|
||||
const tokenizerSuffix = ".tokenizer.json" |
||||
|
||||
type LLM struct { |
||||
base.Base |
||||
|
||||
rwkv *rwkv.RwkvState |
||||
} |
||||
|
||||
func (llm *LLM) Load(opts *pb.ModelOptions) error { |
||||
modelPath := filepath.Dir(opts.Model) |
||||
modelFile := filepath.Base(opts.Model) |
||||
model := rwkv.LoadFiles(opts.Model, filepath.Join(modelPath, modelFile+tokenizerSuffix), uint32(opts.GetThreads())) |
||||
|
||||
if model == nil { |
||||
return fmt.Errorf("could not load model") |
||||
} |
||||
llm.rwkv = model |
||||
return nil |
||||
} |
||||
|
||||
func (llm *LLM) Predict(opts *pb.PredictOptions) (string, error) { |
||||
|
||||
stopWord := "\n" |
||||
if len(opts.StopPrompts) > 0 { |
||||
stopWord = opts.StopPrompts[0] |
||||
} |
||||
|
||||
if err := llm.rwkv.ProcessInput(opts.Prompt); err != nil { |
||||
return "", err |
||||
} |
||||
|
||||
response := llm.rwkv.GenerateResponse(int(opts.Tokens), stopWord, float32(opts.Temperature), float32(opts.TopP), nil) |
||||
|
||||
return response, nil |
||||
} |
||||
|
||||
func (llm *LLM) PredictStream(opts *pb.PredictOptions, results chan string) error { |
||||
go func() { |
||||
|
||||
stopWord := "\n" |
||||
if len(opts.StopPrompts) > 0 { |
||||
stopWord = opts.StopPrompts[0] |
||||
} |
||||
|
||||
if err := llm.rwkv.ProcessInput(opts.Prompt); err != nil { |
||||
fmt.Println("Error processing input: ", err) |
||||
return |
||||
} |
||||
|
||||
llm.rwkv.GenerateResponse(int(opts.Tokens), stopWord, float32(opts.Temperature), float32(opts.TopP), func(s string) bool { |
||||
results <- s |
||||
return true |
||||
}) |
||||
close(results) |
||||
}() |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,43 @@ |
||||
package transformers |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/go-skynet/LocalAI/pkg/grpc/base" |
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
|
||||
transformers "github.com/go-skynet/go-ggml-transformers.cpp" |
||||
) |
||||
|
||||
type Falcon struct { |
||||
base.Base |
||||
|
||||
falcon *transformers.Falcon |
||||
} |
||||
|
||||
func (llm *Falcon) Load(opts *pb.ModelOptions) error { |
||||
model, err := transformers.NewFalcon(opts.Model) |
||||
llm.falcon = model |
||||
return err |
||||
} |
||||
|
||||
func (llm *Falcon) Predict(opts *pb.PredictOptions) (string, error) { |
||||
return llm.falcon.Predict(opts.Prompt, buildPredictOptions(opts)...) |
||||
} |
||||
|
||||
// fallback to Predict
|
||||
func (llm *Falcon) PredictStream(opts *pb.PredictOptions, results chan string) error { |
||||
go func() { |
||||
res, err := llm.falcon.Predict(opts.Prompt, buildPredictOptions(opts)...) |
||||
|
||||
if err != nil { |
||||
fmt.Println("err: ", err) |
||||
} |
||||
results <- res |
||||
close(results) |
||||
}() |
||||
|
||||
return nil |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,385 @@ |
||||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
// versions:
|
||||
// - protoc-gen-go-grpc v1.2.0
|
||||
// - protoc v3.15.8
|
||||
// source: pkg/grpc/proto/backend.proto
|
||||
|
||||
package proto |
||||
|
||||
import ( |
||||
context "context" |
||||
grpc "google.golang.org/grpc" |
||||
codes "google.golang.org/grpc/codes" |
||||
status "google.golang.org/grpc/status" |
||||
) |
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.32.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion7 |
||||
|
||||
// BackendClient is the client API for Backend service.
|
||||
//
|
||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||
type BackendClient interface { |
||||
Health(ctx context.Context, in *HealthMessage, opts ...grpc.CallOption) (*Reply, error) |
||||
Predict(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (*Reply, error) |
||||
LoadModel(ctx context.Context, in *ModelOptions, opts ...grpc.CallOption) (*Result, error) |
||||
PredictStream(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (Backend_PredictStreamClient, error) |
||||
Embedding(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (*EmbeddingResult, error) |
||||
GenerateImage(ctx context.Context, in *GenerateImageRequest, opts ...grpc.CallOption) (*Result, error) |
||||
AudioTranscription(ctx context.Context, in *TranscriptRequest, opts ...grpc.CallOption) (*TranscriptResult, error) |
||||
TTS(ctx context.Context, in *TTSRequest, opts ...grpc.CallOption) (*Result, error) |
||||
} |
||||
|
||||
type backendClient struct { |
||||
cc grpc.ClientConnInterface |
||||
} |
||||
|
||||
func NewBackendClient(cc grpc.ClientConnInterface) BackendClient { |
||||
return &backendClient{cc} |
||||
} |
||||
|
||||
func (c *backendClient) Health(ctx context.Context, in *HealthMessage, opts ...grpc.CallOption) (*Reply, error) { |
||||
out := new(Reply) |
||||
err := c.cc.Invoke(ctx, "/backend.Backend/Health", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *backendClient) Predict(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (*Reply, error) { |
||||
out := new(Reply) |
||||
err := c.cc.Invoke(ctx, "/backend.Backend/Predict", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *backendClient) LoadModel(ctx context.Context, in *ModelOptions, opts ...grpc.CallOption) (*Result, error) { |
||||
out := new(Result) |
||||
err := c.cc.Invoke(ctx, "/backend.Backend/LoadModel", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *backendClient) PredictStream(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (Backend_PredictStreamClient, error) { |
||||
stream, err := c.cc.NewStream(ctx, &Backend_ServiceDesc.Streams[0], "/backend.Backend/PredictStream", opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
x := &backendPredictStreamClient{stream} |
||||
if err := x.ClientStream.SendMsg(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if err := x.ClientStream.CloseSend(); err != nil { |
||||
return nil, err |
||||
} |
||||
return x, nil |
||||
} |
||||
|
||||
type Backend_PredictStreamClient interface { |
||||
Recv() (*Reply, error) |
||||
grpc.ClientStream |
||||
} |
||||
|
||||
type backendPredictStreamClient struct { |
||||
grpc.ClientStream |
||||
} |
||||
|
||||
func (x *backendPredictStreamClient) Recv() (*Reply, error) { |
||||
m := new(Reply) |
||||
if err := x.ClientStream.RecvMsg(m); err != nil { |
||||
return nil, err |
||||
} |
||||
return m, nil |
||||
} |
||||
|
||||
func (c *backendClient) Embedding(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (*EmbeddingResult, error) { |
||||
out := new(EmbeddingResult) |
||||
err := c.cc.Invoke(ctx, "/backend.Backend/Embedding", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *backendClient) GenerateImage(ctx context.Context, in *GenerateImageRequest, opts ...grpc.CallOption) (*Result, error) { |
||||
out := new(Result) |
||||
err := c.cc.Invoke(ctx, "/backend.Backend/GenerateImage", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *backendClient) AudioTranscription(ctx context.Context, in *TranscriptRequest, opts ...grpc.CallOption) (*TranscriptResult, error) { |
||||
out := new(TranscriptResult) |
||||
err := c.cc.Invoke(ctx, "/backend.Backend/AudioTranscription", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *backendClient) TTS(ctx context.Context, in *TTSRequest, opts ...grpc.CallOption) (*Result, error) { |
||||
out := new(Result) |
||||
err := c.cc.Invoke(ctx, "/backend.Backend/TTS", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
// BackendServer is the server API for Backend service.
|
||||
// All implementations must embed UnimplementedBackendServer
|
||||
// for forward compatibility
|
||||
type BackendServer interface { |
||||
Health(context.Context, *HealthMessage) (*Reply, error) |
||||
Predict(context.Context, *PredictOptions) (*Reply, error) |
||||
LoadModel(context.Context, *ModelOptions) (*Result, error) |
||||
PredictStream(*PredictOptions, Backend_PredictStreamServer) error |
||||
Embedding(context.Context, *PredictOptions) (*EmbeddingResult, error) |
||||
GenerateImage(context.Context, *GenerateImageRequest) (*Result, error) |
||||
AudioTranscription(context.Context, *TranscriptRequest) (*TranscriptResult, error) |
||||
TTS(context.Context, *TTSRequest) (*Result, error) |
||||
mustEmbedUnimplementedBackendServer() |
||||
} |
||||
|
||||
// UnimplementedBackendServer must be embedded to have forward compatible implementations.
|
||||
type UnimplementedBackendServer struct { |
||||
} |
||||
|
||||
func (UnimplementedBackendServer) Health(context.Context, *HealthMessage) (*Reply, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method Health not implemented") |
||||
} |
||||
func (UnimplementedBackendServer) Predict(context.Context, *PredictOptions) (*Reply, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method Predict not implemented") |
||||
} |
||||
func (UnimplementedBackendServer) LoadModel(context.Context, *ModelOptions) (*Result, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method LoadModel not implemented") |
||||
} |
||||
func (UnimplementedBackendServer) PredictStream(*PredictOptions, Backend_PredictStreamServer) error { |
||||
return status.Errorf(codes.Unimplemented, "method PredictStream not implemented") |
||||
} |
||||
func (UnimplementedBackendServer) Embedding(context.Context, *PredictOptions) (*EmbeddingResult, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method Embedding not implemented") |
||||
} |
||||
func (UnimplementedBackendServer) GenerateImage(context.Context, *GenerateImageRequest) (*Result, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method GenerateImage not implemented") |
||||
} |
||||
func (UnimplementedBackendServer) AudioTranscription(context.Context, *TranscriptRequest) (*TranscriptResult, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method AudioTranscription not implemented") |
||||
} |
||||
func (UnimplementedBackendServer) TTS(context.Context, *TTSRequest) (*Result, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method TTS not implemented") |
||||
} |
||||
func (UnimplementedBackendServer) mustEmbedUnimplementedBackendServer() {} |
||||
|
||||
// UnsafeBackendServer may be embedded to opt out of forward compatibility for this service.
|
||||
// Use of this interface is not recommended, as added methods to BackendServer will
|
||||
// result in compilation errors.
|
||||
type UnsafeBackendServer interface { |
||||
mustEmbedUnimplementedBackendServer() |
||||
} |
||||
|
||||
func RegisterBackendServer(s grpc.ServiceRegistrar, srv BackendServer) { |
||||
s.RegisterService(&Backend_ServiceDesc, srv) |
||||
} |
||||
|
||||
func _Backend_Health_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(HealthMessage) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(BackendServer).Health(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/backend.Backend/Health", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(BackendServer).Health(ctx, req.(*HealthMessage)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _Backend_Predict_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(PredictOptions) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(BackendServer).Predict(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/backend.Backend/Predict", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(BackendServer).Predict(ctx, req.(*PredictOptions)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _Backend_LoadModel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(ModelOptions) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(BackendServer).LoadModel(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/backend.Backend/LoadModel", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(BackendServer).LoadModel(ctx, req.(*ModelOptions)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _Backend_PredictStream_Handler(srv interface{}, stream grpc.ServerStream) error { |
||||
m := new(PredictOptions) |
||||
if err := stream.RecvMsg(m); err != nil { |
||||
return err |
||||
} |
||||
return srv.(BackendServer).PredictStream(m, &backendPredictStreamServer{stream}) |
||||
} |
||||
|
||||
type Backend_PredictStreamServer interface { |
||||
Send(*Reply) error |
||||
grpc.ServerStream |
||||
} |
||||
|
||||
type backendPredictStreamServer struct { |
||||
grpc.ServerStream |
||||
} |
||||
|
||||
func (x *backendPredictStreamServer) Send(m *Reply) error { |
||||
return x.ServerStream.SendMsg(m) |
||||
} |
||||
|
||||
func _Backend_Embedding_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(PredictOptions) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(BackendServer).Embedding(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/backend.Backend/Embedding", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(BackendServer).Embedding(ctx, req.(*PredictOptions)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _Backend_GenerateImage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(GenerateImageRequest) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(BackendServer).GenerateImage(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/backend.Backend/GenerateImage", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(BackendServer).GenerateImage(ctx, req.(*GenerateImageRequest)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _Backend_AudioTranscription_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(TranscriptRequest) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(BackendServer).AudioTranscription(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/backend.Backend/AudioTranscription", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(BackendServer).AudioTranscription(ctx, req.(*TranscriptRequest)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _Backend_TTS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(TTSRequest) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(BackendServer).TTS(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/backend.Backend/TTS", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(BackendServer).TTS(ctx, req.(*TTSRequest)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
// Backend_ServiceDesc is the grpc.ServiceDesc for Backend service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
var Backend_ServiceDesc = grpc.ServiceDesc{ |
||||
ServiceName: "backend.Backend", |
||||
HandlerType: (*BackendServer)(nil), |
||||
Methods: []grpc.MethodDesc{ |
||||
{ |
||||
MethodName: "Health", |
||||
Handler: _Backend_Health_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "Predict", |
||||
Handler: _Backend_Predict_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "LoadModel", |
||||
Handler: _Backend_LoadModel_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "Embedding", |
||||
Handler: _Backend_Embedding_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "GenerateImage", |
||||
Handler: _Backend_GenerateImage_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "AudioTranscription", |
||||
Handler: _Backend_AudioTranscription_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "TTS", |
||||
Handler: _Backend_TTS_Handler, |
||||
}, |
||||
}, |
||||
Streams: []grpc.StreamDesc{ |
||||
{ |
||||
StreamName: "PredictStream", |
||||
Handler: _Backend_PredictStream_Handler, |
||||
ServerStreams: true, |
||||
}, |
||||
}, |
||||
Metadata: "pkg/grpc/proto/backend.proto", |
||||
} |
||||
@ -1,969 +0,0 @@ |
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.15.8
|
||||
// source: pkg/grpc/proto/llmserver.proto
|
||||
|
||||
package proto |
||||
|
||||
import ( |
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect" |
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl" |
||||
reflect "reflect" |
||||
sync "sync" |
||||
) |
||||
|
||||
const ( |
||||
// Verify that this generated code is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) |
||||
// Verify that runtime/protoimpl is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) |
||||
) |
||||
|
||||
type HealthMessage struct { |
||||
state protoimpl.MessageState |
||||
sizeCache protoimpl.SizeCache |
||||
unknownFields protoimpl.UnknownFields |
||||
} |
||||
|
||||
func (x *HealthMessage) Reset() { |
||||
*x = HealthMessage{} |
||||
if protoimpl.UnsafeEnabled { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[0] |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
} |
||||
|
||||
func (x *HealthMessage) String() string { |
||||
return protoimpl.X.MessageStringOf(x) |
||||
} |
||||
|
||||
func (*HealthMessage) ProtoMessage() {} |
||||
|
||||
func (x *HealthMessage) ProtoReflect() protoreflect.Message { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[0] |
||||
if protoimpl.UnsafeEnabled && x != nil { |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
if ms.LoadMessageInfo() == nil { |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
return ms |
||||
} |
||||
return mi.MessageOf(x) |
||||
} |
||||
|
||||
// Deprecated: Use HealthMessage.ProtoReflect.Descriptor instead.
|
||||
func (*HealthMessage) Descriptor() ([]byte, []int) { |
||||
return file_pkg_grpc_proto_llmserver_proto_rawDescGZIP(), []int{0} |
||||
} |
||||
|
||||
// The request message containing the user's name.
|
||||
type PredictOptions struct { |
||||
state protoimpl.MessageState |
||||
sizeCache protoimpl.SizeCache |
||||
unknownFields protoimpl.UnknownFields |
||||
|
||||
Prompt string `protobuf:"bytes,1,opt,name=Prompt,proto3" json:"Prompt,omitempty"` |
||||
Seed int32 `protobuf:"varint,2,opt,name=Seed,proto3" json:"Seed,omitempty"` |
||||
Threads int32 `protobuf:"varint,3,opt,name=Threads,proto3" json:"Threads,omitempty"` |
||||
Tokens int32 `protobuf:"varint,4,opt,name=Tokens,proto3" json:"Tokens,omitempty"` |
||||
TopK int32 `protobuf:"varint,5,opt,name=TopK,proto3" json:"TopK,omitempty"` |
||||
Repeat int32 `protobuf:"varint,6,opt,name=Repeat,proto3" json:"Repeat,omitempty"` |
||||
Batch int32 `protobuf:"varint,7,opt,name=Batch,proto3" json:"Batch,omitempty"` |
||||
NKeep int32 `protobuf:"varint,8,opt,name=NKeep,proto3" json:"NKeep,omitempty"` |
||||
Temperature float32 `protobuf:"fixed32,9,opt,name=Temperature,proto3" json:"Temperature,omitempty"` |
||||
Penalty float32 `protobuf:"fixed32,10,opt,name=Penalty,proto3" json:"Penalty,omitempty"` |
||||
F16KV bool `protobuf:"varint,11,opt,name=F16KV,proto3" json:"F16KV,omitempty"` |
||||
DebugMode bool `protobuf:"varint,12,opt,name=DebugMode,proto3" json:"DebugMode,omitempty"` |
||||
StopPrompts []string `protobuf:"bytes,13,rep,name=StopPrompts,proto3" json:"StopPrompts,omitempty"` |
||||
IgnoreEOS bool `protobuf:"varint,14,opt,name=IgnoreEOS,proto3" json:"IgnoreEOS,omitempty"` |
||||
TailFreeSamplingZ float32 `protobuf:"fixed32,15,opt,name=TailFreeSamplingZ,proto3" json:"TailFreeSamplingZ,omitempty"` |
||||
TypicalP float32 `protobuf:"fixed32,16,opt,name=TypicalP,proto3" json:"TypicalP,omitempty"` |
||||
FrequencyPenalty float32 `protobuf:"fixed32,17,opt,name=FrequencyPenalty,proto3" json:"FrequencyPenalty,omitempty"` |
||||
PresencePenalty float32 `protobuf:"fixed32,18,opt,name=PresencePenalty,proto3" json:"PresencePenalty,omitempty"` |
||||
Mirostat int32 `protobuf:"varint,19,opt,name=Mirostat,proto3" json:"Mirostat,omitempty"` |
||||
MirostatETA float32 `protobuf:"fixed32,20,opt,name=MirostatETA,proto3" json:"MirostatETA,omitempty"` |
||||
MirostatTAU float32 `protobuf:"fixed32,21,opt,name=MirostatTAU,proto3" json:"MirostatTAU,omitempty"` |
||||
PenalizeNL bool `protobuf:"varint,22,opt,name=PenalizeNL,proto3" json:"PenalizeNL,omitempty"` |
||||
LogitBias string `protobuf:"bytes,23,opt,name=LogitBias,proto3" json:"LogitBias,omitempty"` |
||||
MLock bool `protobuf:"varint,25,opt,name=MLock,proto3" json:"MLock,omitempty"` |
||||
MMap bool `protobuf:"varint,26,opt,name=MMap,proto3" json:"MMap,omitempty"` |
||||
PromptCacheAll bool `protobuf:"varint,27,opt,name=PromptCacheAll,proto3" json:"PromptCacheAll,omitempty"` |
||||
PromptCacheRO bool `protobuf:"varint,28,opt,name=PromptCacheRO,proto3" json:"PromptCacheRO,omitempty"` |
||||
Grammar string `protobuf:"bytes,29,opt,name=Grammar,proto3" json:"Grammar,omitempty"` |
||||
MainGPU string `protobuf:"bytes,30,opt,name=MainGPU,proto3" json:"MainGPU,omitempty"` |
||||
TensorSplit string `protobuf:"bytes,31,opt,name=TensorSplit,proto3" json:"TensorSplit,omitempty"` |
||||
TopP float32 `protobuf:"fixed32,32,opt,name=TopP,proto3" json:"TopP,omitempty"` |
||||
PromptCachePath string `protobuf:"bytes,33,opt,name=PromptCachePath,proto3" json:"PromptCachePath,omitempty"` |
||||
Debug bool `protobuf:"varint,34,opt,name=Debug,proto3" json:"Debug,omitempty"` |
||||
EmbeddingTokens []int32 `protobuf:"varint,35,rep,packed,name=EmbeddingTokens,proto3" json:"EmbeddingTokens,omitempty"` |
||||
Embeddings string `protobuf:"bytes,36,opt,name=Embeddings,proto3" json:"Embeddings,omitempty"` |
||||
} |
||||
|
||||
func (x *PredictOptions) Reset() { |
||||
*x = PredictOptions{} |
||||
if protoimpl.UnsafeEnabled { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[1] |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
} |
||||
|
||||
func (x *PredictOptions) String() string { |
||||
return protoimpl.X.MessageStringOf(x) |
||||
} |
||||
|
||||
func (*PredictOptions) ProtoMessage() {} |
||||
|
||||
func (x *PredictOptions) ProtoReflect() protoreflect.Message { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[1] |
||||
if protoimpl.UnsafeEnabled && x != nil { |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
if ms.LoadMessageInfo() == nil { |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
return ms |
||||
} |
||||
return mi.MessageOf(x) |
||||
} |
||||
|
||||
// Deprecated: Use PredictOptions.ProtoReflect.Descriptor instead.
|
||||
func (*PredictOptions) Descriptor() ([]byte, []int) { |
||||
return file_pkg_grpc_proto_llmserver_proto_rawDescGZIP(), []int{1} |
||||
} |
||||
|
||||
func (x *PredictOptions) GetPrompt() string { |
||||
if x != nil { |
||||
return x.Prompt |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *PredictOptions) GetSeed() int32 { |
||||
if x != nil { |
||||
return x.Seed |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetThreads() int32 { |
||||
if x != nil { |
||||
return x.Threads |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetTokens() int32 { |
||||
if x != nil { |
||||
return x.Tokens |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetTopK() int32 { |
||||
if x != nil { |
||||
return x.TopK |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetRepeat() int32 { |
||||
if x != nil { |
||||
return x.Repeat |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetBatch() int32 { |
||||
if x != nil { |
||||
return x.Batch |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetNKeep() int32 { |
||||
if x != nil { |
||||
return x.NKeep |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetTemperature() float32 { |
||||
if x != nil { |
||||
return x.Temperature |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetPenalty() float32 { |
||||
if x != nil { |
||||
return x.Penalty |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetF16KV() bool { |
||||
if x != nil { |
||||
return x.F16KV |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetDebugMode() bool { |
||||
if x != nil { |
||||
return x.DebugMode |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetStopPrompts() []string { |
||||
if x != nil { |
||||
return x.StopPrompts |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (x *PredictOptions) GetIgnoreEOS() bool { |
||||
if x != nil { |
||||
return x.IgnoreEOS |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetTailFreeSamplingZ() float32 { |
||||
if x != nil { |
||||
return x.TailFreeSamplingZ |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetTypicalP() float32 { |
||||
if x != nil { |
||||
return x.TypicalP |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetFrequencyPenalty() float32 { |
||||
if x != nil { |
||||
return x.FrequencyPenalty |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetPresencePenalty() float32 { |
||||
if x != nil { |
||||
return x.PresencePenalty |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetMirostat() int32 { |
||||
if x != nil { |
||||
return x.Mirostat |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetMirostatETA() float32 { |
||||
if x != nil { |
||||
return x.MirostatETA |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetMirostatTAU() float32 { |
||||
if x != nil { |
||||
return x.MirostatTAU |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetPenalizeNL() bool { |
||||
if x != nil { |
||||
return x.PenalizeNL |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetLogitBias() string { |
||||
if x != nil { |
||||
return x.LogitBias |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *PredictOptions) GetMLock() bool { |
||||
if x != nil { |
||||
return x.MLock |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetMMap() bool { |
||||
if x != nil { |
||||
return x.MMap |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetPromptCacheAll() bool { |
||||
if x != nil { |
||||
return x.PromptCacheAll |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetPromptCacheRO() bool { |
||||
if x != nil { |
||||
return x.PromptCacheRO |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetGrammar() string { |
||||
if x != nil { |
||||
return x.Grammar |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *PredictOptions) GetMainGPU() string { |
||||
if x != nil { |
||||
return x.MainGPU |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *PredictOptions) GetTensorSplit() string { |
||||
if x != nil { |
||||
return x.TensorSplit |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *PredictOptions) GetTopP() float32 { |
||||
if x != nil { |
||||
return x.TopP |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *PredictOptions) GetPromptCachePath() string { |
||||
if x != nil { |
||||
return x.PromptCachePath |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *PredictOptions) GetDebug() bool { |
||||
if x != nil { |
||||
return x.Debug |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *PredictOptions) GetEmbeddingTokens() []int32 { |
||||
if x != nil { |
||||
return x.EmbeddingTokens |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (x *PredictOptions) GetEmbeddings() string { |
||||
if x != nil { |
||||
return x.Embeddings |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
// The response message containing the result
|
||||
type Reply struct { |
||||
state protoimpl.MessageState |
||||
sizeCache protoimpl.SizeCache |
||||
unknownFields protoimpl.UnknownFields |
||||
|
||||
Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` |
||||
} |
||||
|
||||
func (x *Reply) Reset() { |
||||
*x = Reply{} |
||||
if protoimpl.UnsafeEnabled { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[2] |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
} |
||||
|
||||
func (x *Reply) String() string { |
||||
return protoimpl.X.MessageStringOf(x) |
||||
} |
||||
|
||||
func (*Reply) ProtoMessage() {} |
||||
|
||||
func (x *Reply) ProtoReflect() protoreflect.Message { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[2] |
||||
if protoimpl.UnsafeEnabled && x != nil { |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
if ms.LoadMessageInfo() == nil { |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
return ms |
||||
} |
||||
return mi.MessageOf(x) |
||||
} |
||||
|
||||
// Deprecated: Use Reply.ProtoReflect.Descriptor instead.
|
||||
func (*Reply) Descriptor() ([]byte, []int) { |
||||
return file_pkg_grpc_proto_llmserver_proto_rawDescGZIP(), []int{2} |
||||
} |
||||
|
||||
func (x *Reply) GetMessage() string { |
||||
if x != nil { |
||||
return x.Message |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
type ModelOptions struct { |
||||
state protoimpl.MessageState |
||||
sizeCache protoimpl.SizeCache |
||||
unknownFields protoimpl.UnknownFields |
||||
|
||||
Model string `protobuf:"bytes,1,opt,name=Model,proto3" json:"Model,omitempty"` |
||||
ContextSize int32 `protobuf:"varint,2,opt,name=ContextSize,proto3" json:"ContextSize,omitempty"` |
||||
Seed int32 `protobuf:"varint,3,opt,name=Seed,proto3" json:"Seed,omitempty"` |
||||
NBatch int32 `protobuf:"varint,4,opt,name=NBatch,proto3" json:"NBatch,omitempty"` |
||||
F16Memory bool `protobuf:"varint,5,opt,name=F16Memory,proto3" json:"F16Memory,omitempty"` |
||||
MLock bool `protobuf:"varint,6,opt,name=MLock,proto3" json:"MLock,omitempty"` |
||||
MMap bool `protobuf:"varint,7,opt,name=MMap,proto3" json:"MMap,omitempty"` |
||||
VocabOnly bool `protobuf:"varint,8,opt,name=VocabOnly,proto3" json:"VocabOnly,omitempty"` |
||||
LowVRAM bool `protobuf:"varint,9,opt,name=LowVRAM,proto3" json:"LowVRAM,omitempty"` |
||||
Embeddings bool `protobuf:"varint,10,opt,name=Embeddings,proto3" json:"Embeddings,omitempty"` |
||||
NUMA bool `protobuf:"varint,11,opt,name=NUMA,proto3" json:"NUMA,omitempty"` |
||||
NGPULayers int32 `protobuf:"varint,12,opt,name=NGPULayers,proto3" json:"NGPULayers,omitempty"` |
||||
MainGPU string `protobuf:"bytes,13,opt,name=MainGPU,proto3" json:"MainGPU,omitempty"` |
||||
TensorSplit string `protobuf:"bytes,14,opt,name=TensorSplit,proto3" json:"TensorSplit,omitempty"` |
||||
Threads int32 `protobuf:"varint,15,opt,name=Threads,proto3" json:"Threads,omitempty"` |
||||
LibrarySearchPath string `protobuf:"bytes,16,opt,name=LibrarySearchPath,proto3" json:"LibrarySearchPath,omitempty"` |
||||
} |
||||
|
||||
func (x *ModelOptions) Reset() { |
||||
*x = ModelOptions{} |
||||
if protoimpl.UnsafeEnabled { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[3] |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
} |
||||
|
||||
func (x *ModelOptions) String() string { |
||||
return protoimpl.X.MessageStringOf(x) |
||||
} |
||||
|
||||
func (*ModelOptions) ProtoMessage() {} |
||||
|
||||
func (x *ModelOptions) ProtoReflect() protoreflect.Message { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[3] |
||||
if protoimpl.UnsafeEnabled && x != nil { |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
if ms.LoadMessageInfo() == nil { |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
return ms |
||||
} |
||||
return mi.MessageOf(x) |
||||
} |
||||
|
||||
// Deprecated: Use ModelOptions.ProtoReflect.Descriptor instead.
|
||||
func (*ModelOptions) Descriptor() ([]byte, []int) { |
||||
return file_pkg_grpc_proto_llmserver_proto_rawDescGZIP(), []int{3} |
||||
} |
||||
|
||||
func (x *ModelOptions) GetModel() string { |
||||
if x != nil { |
||||
return x.Model |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *ModelOptions) GetContextSize() int32 { |
||||
if x != nil { |
||||
return x.ContextSize |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *ModelOptions) GetSeed() int32 { |
||||
if x != nil { |
||||
return x.Seed |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *ModelOptions) GetNBatch() int32 { |
||||
if x != nil { |
||||
return x.NBatch |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *ModelOptions) GetF16Memory() bool { |
||||
if x != nil { |
||||
return x.F16Memory |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *ModelOptions) GetMLock() bool { |
||||
if x != nil { |
||||
return x.MLock |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *ModelOptions) GetMMap() bool { |
||||
if x != nil { |
||||
return x.MMap |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *ModelOptions) GetVocabOnly() bool { |
||||
if x != nil { |
||||
return x.VocabOnly |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *ModelOptions) GetLowVRAM() bool { |
||||
if x != nil { |
||||
return x.LowVRAM |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *ModelOptions) GetEmbeddings() bool { |
||||
if x != nil { |
||||
return x.Embeddings |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *ModelOptions) GetNUMA() bool { |
||||
if x != nil { |
||||
return x.NUMA |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (x *ModelOptions) GetNGPULayers() int32 { |
||||
if x != nil { |
||||
return x.NGPULayers |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *ModelOptions) GetMainGPU() string { |
||||
if x != nil { |
||||
return x.MainGPU |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *ModelOptions) GetTensorSplit() string { |
||||
if x != nil { |
||||
return x.TensorSplit |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *ModelOptions) GetThreads() int32 { |
||||
if x != nil { |
||||
return x.Threads |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (x *ModelOptions) GetLibrarySearchPath() string { |
||||
if x != nil { |
||||
return x.LibrarySearchPath |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
type Result struct { |
||||
state protoimpl.MessageState |
||||
sizeCache protoimpl.SizeCache |
||||
unknownFields protoimpl.UnknownFields |
||||
|
||||
Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` |
||||
Success bool `protobuf:"varint,2,opt,name=success,proto3" json:"success,omitempty"` |
||||
} |
||||
|
||||
func (x *Result) Reset() { |
||||
*x = Result{} |
||||
if protoimpl.UnsafeEnabled { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[4] |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
} |
||||
|
||||
func (x *Result) String() string { |
||||
return protoimpl.X.MessageStringOf(x) |
||||
} |
||||
|
||||
func (*Result) ProtoMessage() {} |
||||
|
||||
func (x *Result) ProtoReflect() protoreflect.Message { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[4] |
||||
if protoimpl.UnsafeEnabled && x != nil { |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
if ms.LoadMessageInfo() == nil { |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
return ms |
||||
} |
||||
return mi.MessageOf(x) |
||||
} |
||||
|
||||
// Deprecated: Use Result.ProtoReflect.Descriptor instead.
|
||||
func (*Result) Descriptor() ([]byte, []int) { |
||||
return file_pkg_grpc_proto_llmserver_proto_rawDescGZIP(), []int{4} |
||||
} |
||||
|
||||
func (x *Result) GetMessage() string { |
||||
if x != nil { |
||||
return x.Message |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (x *Result) GetSuccess() bool { |
||||
if x != nil { |
||||
return x.Success |
||||
} |
||||
return false |
||||
} |
||||
|
||||
type EmbeddingResult struct { |
||||
state protoimpl.MessageState |
||||
sizeCache protoimpl.SizeCache |
||||
unknownFields protoimpl.UnknownFields |
||||
|
||||
Embeddings []float32 `protobuf:"fixed32,1,rep,packed,name=embeddings,proto3" json:"embeddings,omitempty"` |
||||
} |
||||
|
||||
func (x *EmbeddingResult) Reset() { |
||||
*x = EmbeddingResult{} |
||||
if protoimpl.UnsafeEnabled { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[5] |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
} |
||||
|
||||
func (x *EmbeddingResult) String() string { |
||||
return protoimpl.X.MessageStringOf(x) |
||||
} |
||||
|
||||
func (*EmbeddingResult) ProtoMessage() {} |
||||
|
||||
func (x *EmbeddingResult) ProtoReflect() protoreflect.Message { |
||||
mi := &file_pkg_grpc_proto_llmserver_proto_msgTypes[5] |
||||
if protoimpl.UnsafeEnabled && x != nil { |
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
||||
if ms.LoadMessageInfo() == nil { |
||||
ms.StoreMessageInfo(mi) |
||||
} |
||||
return ms |
||||
} |
||||
return mi.MessageOf(x) |
||||
} |
||||
|
||||
// Deprecated: Use EmbeddingResult.ProtoReflect.Descriptor instead.
|
||||
func (*EmbeddingResult) Descriptor() ([]byte, []int) { |
||||
return file_pkg_grpc_proto_llmserver_proto_rawDescGZIP(), []int{5} |
||||
} |
||||
|
||||
func (x *EmbeddingResult) GetEmbeddings() []float32 { |
||||
if x != nil { |
||||
return x.Embeddings |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
var File_pkg_grpc_proto_llmserver_proto protoreflect.FileDescriptor |
||||
|
||||
var file_pkg_grpc_proto_llmserver_proto_rawDesc = []byte{ |
||||
0x0a, 0x1e, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, |
||||
0x2f, 0x6c, 0x6c, 0x6d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, |
||||
0x12, 0x03, 0x6c, 0x6c, 0x6d, 0x22, 0x0f, 0x0a, 0x0d, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x4d, |
||||
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xa0, 0x08, 0x0a, 0x0e, 0x50, 0x72, 0x65, 0x64, 0x69, |
||||
0x63, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x72, 0x6f, |
||||
0x6d, 0x70, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x72, 0x6f, 0x6d, 0x70, |
||||
0x74, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x65, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, |
||||
0x04, 0x53, 0x65, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, |
||||
0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, |
||||
0x16, 0x0a, 0x06, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, |
||||
0x06, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x6f, 0x70, 0x4b, 0x18, |
||||
0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x54, 0x6f, 0x70, 0x4b, 0x12, 0x16, 0x0a, 0x06, 0x52, |
||||
0x65, 0x70, 0x65, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x52, 0x65, 0x70, |
||||
0x65, 0x61, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x42, 0x61, 0x74, 0x63, 0x68, 0x18, 0x07, 0x20, 0x01, |
||||
0x28, 0x05, 0x52, 0x05, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x4b, 0x65, |
||||
0x65, 0x70, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x4e, 0x4b, 0x65, 0x65, 0x70, 0x12, |
||||
0x20, 0x0a, 0x0b, 0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x09, |
||||
0x20, 0x01, 0x28, 0x02, 0x52, 0x0b, 0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, |
||||
0x65, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x18, 0x0a, 0x20, 0x01, |
||||
0x28, 0x02, 0x52, 0x07, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x46, |
||||
0x31, 0x36, 0x4b, 0x56, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x46, 0x31, 0x36, 0x4b, |
||||
0x56, 0x12, 0x1c, 0x0a, 0x09, 0x44, 0x65, 0x62, 0x75, 0x67, 0x4d, 0x6f, 0x64, 0x65, 0x18, 0x0c, |
||||
0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x44, 0x65, 0x62, 0x75, 0x67, 0x4d, 0x6f, 0x64, 0x65, 0x12, |
||||
0x20, 0x0a, 0x0b, 0x53, 0x74, 0x6f, 0x70, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x73, 0x18, 0x0d, |
||||
0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x74, 0x6f, 0x70, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, |
||||
0x73, 0x12, 0x1c, 0x0a, 0x09, 0x49, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x4f, 0x53, 0x18, 0x0e, |
||||
0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x49, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x4f, 0x53, 0x12, |
||||
0x2c, 0x0a, 0x11, 0x54, 0x61, 0x69, 0x6c, 0x46, 0x72, 0x65, 0x65, 0x53, 0x61, 0x6d, 0x70, 0x6c, |
||||
0x69, 0x6e, 0x67, 0x5a, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x02, 0x52, 0x11, 0x54, 0x61, 0x69, 0x6c, |
||||
0x46, 0x72, 0x65, 0x65, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x69, 0x6e, 0x67, 0x5a, 0x12, 0x1a, 0x0a, |
||||
0x08, 0x54, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x18, 0x10, 0x20, 0x01, 0x28, 0x02, 0x52, |
||||
0x08, 0x54, 0x79, 0x70, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x12, 0x2a, 0x0a, 0x10, 0x46, 0x72, 0x65, |
||||
0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x18, 0x11, 0x20, |
||||
0x01, 0x28, 0x02, 0x52, 0x10, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x50, 0x65, |
||||
0x6e, 0x61, 0x6c, 0x74, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, |
||||
0x65, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x18, 0x12, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0f, |
||||
0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x74, 0x79, 0x12, |
||||
0x1a, 0x0a, 0x08, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x18, 0x13, 0x20, 0x01, 0x28, |
||||
0x05, 0x52, 0x08, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x4d, |
||||
0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x45, 0x54, 0x41, 0x18, 0x14, 0x20, 0x01, 0x28, 0x02, |
||||
0x52, 0x0b, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x45, 0x54, 0x41, 0x12, 0x20, 0x0a, |
||||
0x0b, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x54, 0x41, 0x55, 0x18, 0x15, 0x20, 0x01, |
||||
0x28, 0x02, 0x52, 0x0b, 0x4d, 0x69, 0x72, 0x6f, 0x73, 0x74, 0x61, 0x74, 0x54, 0x41, 0x55, 0x12, |
||||
0x1e, 0x0a, 0x0a, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x4e, 0x4c, 0x18, 0x16, 0x20, |
||||
0x01, 0x28, 0x08, 0x52, 0x0a, 0x50, 0x65, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x4e, 0x4c, 0x12, |
||||
0x1c, 0x0a, 0x09, 0x4c, 0x6f, 0x67, 0x69, 0x74, 0x42, 0x69, 0x61, 0x73, 0x18, 0x17, 0x20, 0x01, |
||||
0x28, 0x09, 0x52, 0x09, 0x4c, 0x6f, 0x67, 0x69, 0x74, 0x42, 0x69, 0x61, 0x73, 0x12, 0x14, 0x0a, |
||||
0x05, 0x4d, 0x4c, 0x6f, 0x63, 0x6b, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x4d, 0x4c, |
||||
0x6f, 0x63, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x4d, 0x4d, 0x61, 0x70, 0x18, 0x1a, 0x20, 0x01, 0x28, |
||||
0x08, 0x52, 0x04, 0x4d, 0x4d, 0x61, 0x70, 0x12, 0x26, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x6d, 0x70, |
||||
0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x41, 0x6c, 0x6c, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x08, 0x52, |
||||
0x0e, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x41, 0x6c, 0x6c, 0x12, |
||||
0x24, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x4f, |
||||
0x18, 0x1c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, 0x61, |
||||
0x63, 0x68, 0x65, 0x52, 0x4f, 0x12, 0x18, 0x0a, 0x07, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, |
||||
0x18, 0x1d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x47, 0x72, 0x61, 0x6d, 0x6d, 0x61, 0x72, 0x12, |
||||
0x18, 0x0a, 0x07, 0x4d, 0x61, 0x69, 0x6e, 0x47, 0x50, 0x55, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x09, |
||||
0x52, 0x07, 0x4d, 0x61, 0x69, 0x6e, 0x47, 0x50, 0x55, 0x12, 0x20, 0x0a, 0x0b, 0x54, 0x65, 0x6e, |
||||
0x73, 0x6f, 0x72, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, |
||||
0x54, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x54, |
||||
0x6f, 0x70, 0x50, 0x18, 0x20, 0x20, 0x01, 0x28, 0x02, 0x52, 0x04, 0x54, 0x6f, 0x70, 0x50, 0x12, |
||||
0x28, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x43, 0x61, 0x63, 0x68, 0x65, 0x50, 0x61, |
||||
0x74, 0x68, 0x18, 0x21, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, |
||||
0x43, 0x61, 0x63, 0x68, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x44, 0x65, 0x62, |
||||
0x75, 0x67, 0x18, 0x22, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x44, 0x65, 0x62, 0x75, 0x67, 0x12, |
||||
0x28, 0x0a, 0x0f, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, |
||||
0x6e, 0x73, 0x18, 0x23, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0f, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, |
||||
0x69, 0x6e, 0x67, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x45, 0x6d, 0x62, |
||||
0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x45, |
||||
0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x21, 0x0a, 0x05, 0x52, 0x65, 0x70, |
||||
0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, |
||||
0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xca, 0x03, 0x0a, |
||||
0x0c, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x14, 0x0a, |
||||
0x05, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4d, 0x6f, |
||||
0x64, 0x65, 0x6c, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x53, 0x69, |
||||
0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, |
||||
0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x65, 0x65, 0x64, 0x18, 0x03, 0x20, |
||||
0x01, 0x28, 0x05, 0x52, 0x04, 0x53, 0x65, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x4e, 0x42, 0x61, |
||||
0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x4e, 0x42, 0x61, 0x74, 0x63, |
||||
0x68, 0x12, 0x1c, 0x0a, 0x09, 0x46, 0x31, 0x36, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x05, |
||||
0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x46, 0x31, 0x36, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, |
||||
0x14, 0x0a, 0x05, 0x4d, 0x4c, 0x6f, 0x63, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, |
||||
0x4d, 0x4c, 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x4d, 0x4d, 0x61, 0x70, 0x18, 0x07, 0x20, |
||||
0x01, 0x28, 0x08, 0x52, 0x04, 0x4d, 0x4d, 0x61, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x56, 0x6f, 0x63, |
||||
0x61, 0x62, 0x4f, 0x6e, 0x6c, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x56, 0x6f, |
||||
0x63, 0x61, 0x62, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x4c, 0x6f, 0x77, 0x56, 0x52, |
||||
0x41, 0x4d, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x4c, 0x6f, 0x77, 0x56, 0x52, 0x41, |
||||
0x4d, 0x12, 0x1e, 0x0a, 0x0a, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, |
||||
0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, |
||||
0x73, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x55, 0x4d, 0x41, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, |
||||
0x04, 0x4e, 0x55, 0x4d, 0x41, 0x12, 0x1e, 0x0a, 0x0a, 0x4e, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x79, |
||||
0x65, 0x72, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x4e, 0x47, 0x50, 0x55, 0x4c, |
||||
0x61, 0x79, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x61, 0x69, 0x6e, 0x47, 0x50, 0x55, |
||||
0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x61, 0x69, 0x6e, 0x47, 0x50, 0x55, 0x12, |
||||
0x20, 0x0a, 0x0b, 0x54, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x18, 0x0e, |
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x54, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x53, 0x70, 0x6c, 0x69, |
||||
0x74, 0x12, 0x18, 0x0a, 0x07, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x0f, 0x20, 0x01, |
||||
0x28, 0x05, 0x52, 0x07, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x4c, |
||||
0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x61, 0x74, 0x68, |
||||
0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x4c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x53, |
||||
0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x61, 0x74, 0x68, 0x22, 0x3c, 0x0a, 0x06, 0x52, 0x65, 0x73, |
||||
0x75, 0x6c, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, |
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, |
||||
0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, |
||||
0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0x31, 0x0a, 0x0f, 0x45, 0x6d, 0x62, 0x65, 0x64, |
||||
0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x6d, |
||||
0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x02, 0x52, 0x0a, |
||||
0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x32, 0xfe, 0x01, 0x0a, 0x03, 0x4c, |
||||
0x4c, 0x4d, 0x12, 0x2a, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x12, 0x2e, 0x6c, |
||||
0x6c, 0x6d, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, |
||||
0x1a, 0x0a, 0x2e, 0x6c, 0x6c, 0x6d, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x2c, |
||||
0x0a, 0x07, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x12, 0x13, 0x2e, 0x6c, 0x6c, 0x6d, 0x2e, |
||||
0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x0a, |
||||
0x2e, 0x6c, 0x6c, 0x6d, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x09, |
||||
0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x11, 0x2e, 0x6c, 0x6c, 0x6d, 0x2e, |
||||
0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x0b, 0x2e, 0x6c, |
||||
0x6c, 0x6d, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x0d, 0x50, |
||||
0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x13, 0x2e, 0x6c, |
||||
0x6c, 0x6d, 0x2e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, |
||||
0x73, 0x1a, 0x0a, 0x2e, 0x6c, 0x6c, 0x6d, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x30, |
||||
0x01, 0x12, 0x38, 0x0a, 0x09, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x13, |
||||
0x2e, 0x6c, 0x6c, 0x6d, 0x2e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x74, 0x4f, 0x70, 0x74, 0x69, |
||||
0x6f, 0x6e, 0x73, 0x1a, 0x14, 0x2e, 0x6c, 0x6c, 0x6d, 0x2e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, |
||||
0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x00, 0x42, 0x57, 0x0a, 0x1b, 0x69, |
||||
0x6f, 0x2e, 0x73, 0x6b, 0x79, 0x6e, 0x65, 0x74, 0x2e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x61, 0x69, |
||||
0x2e, 0x6c, 0x6c, 0x6d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x42, 0x09, 0x4c, 0x4c, 0x4d, 0x53, |
||||
0x65, 0x72, 0x76, 0x65, 0x72, 0x50, 0x01, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, |
||||
0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x2d, 0x73, 0x6b, 0x79, 0x6e, 0x65, 0x74, 0x2f, 0x4c, 0x6f, |
||||
0x63, 0x61, 0x6c, 0x41, 0x49, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, |
||||
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, |
||||
} |
||||
|
||||
var ( |
||||
file_pkg_grpc_proto_llmserver_proto_rawDescOnce sync.Once |
||||
file_pkg_grpc_proto_llmserver_proto_rawDescData = file_pkg_grpc_proto_llmserver_proto_rawDesc |
||||
) |
||||
|
||||
func file_pkg_grpc_proto_llmserver_proto_rawDescGZIP() []byte { |
||||
file_pkg_grpc_proto_llmserver_proto_rawDescOnce.Do(func() { |
||||
file_pkg_grpc_proto_llmserver_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_grpc_proto_llmserver_proto_rawDescData) |
||||
}) |
||||
return file_pkg_grpc_proto_llmserver_proto_rawDescData |
||||
} |
||||
|
||||
var file_pkg_grpc_proto_llmserver_proto_msgTypes = make([]protoimpl.MessageInfo, 6) |
||||
var file_pkg_grpc_proto_llmserver_proto_goTypes = []interface{}{ |
||||
(*HealthMessage)(nil), // 0: llm.HealthMessage
|
||||
(*PredictOptions)(nil), // 1: llm.PredictOptions
|
||||
(*Reply)(nil), // 2: llm.Reply
|
||||
(*ModelOptions)(nil), // 3: llm.ModelOptions
|
||||
(*Result)(nil), // 4: llm.Result
|
||||
(*EmbeddingResult)(nil), // 5: llm.EmbeddingResult
|
||||
} |
||||
var file_pkg_grpc_proto_llmserver_proto_depIdxs = []int32{ |
||||
0, // 0: llm.LLM.Health:input_type -> llm.HealthMessage
|
||||
1, // 1: llm.LLM.Predict:input_type -> llm.PredictOptions
|
||||
3, // 2: llm.LLM.LoadModel:input_type -> llm.ModelOptions
|
||||
1, // 3: llm.LLM.PredictStream:input_type -> llm.PredictOptions
|
||||
1, // 4: llm.LLM.Embedding:input_type -> llm.PredictOptions
|
||||
2, // 5: llm.LLM.Health:output_type -> llm.Reply
|
||||
2, // 6: llm.LLM.Predict:output_type -> llm.Reply
|
||||
4, // 7: llm.LLM.LoadModel:output_type -> llm.Result
|
||||
2, // 8: llm.LLM.PredictStream:output_type -> llm.Reply
|
||||
5, // 9: llm.LLM.Embedding:output_type -> llm.EmbeddingResult
|
||||
5, // [5:10] is the sub-list for method output_type
|
||||
0, // [0:5] is the sub-list for method input_type
|
||||
0, // [0:0] is the sub-list for extension type_name
|
||||
0, // [0:0] is the sub-list for extension extendee
|
||||
0, // [0:0] is the sub-list for field type_name
|
||||
} |
||||
|
||||
func init() { file_pkg_grpc_proto_llmserver_proto_init() } |
||||
func file_pkg_grpc_proto_llmserver_proto_init() { |
||||
if File_pkg_grpc_proto_llmserver_proto != nil { |
||||
return |
||||
} |
||||
if !protoimpl.UnsafeEnabled { |
||||
file_pkg_grpc_proto_llmserver_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { |
||||
switch v := v.(*HealthMessage); i { |
||||
case 0: |
||||
return &v.state |
||||
case 1: |
||||
return &v.sizeCache |
||||
case 2: |
||||
return &v.unknownFields |
||||
default: |
||||
return nil |
||||
} |
||||
} |
||||
file_pkg_grpc_proto_llmserver_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { |
||||
switch v := v.(*PredictOptions); i { |
||||
case 0: |
||||
return &v.state |
||||
case 1: |
||||
return &v.sizeCache |
||||
case 2: |
||||
return &v.unknownFields |
||||
default: |
||||
return nil |
||||
} |
||||
} |
||||
file_pkg_grpc_proto_llmserver_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { |
||||
switch v := v.(*Reply); i { |
||||
case 0: |
||||
return &v.state |
||||
case 1: |
||||
return &v.sizeCache |
||||
case 2: |
||||
return &v.unknownFields |
||||
default: |
||||
return nil |
||||
} |
||||
} |
||||
file_pkg_grpc_proto_llmserver_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { |
||||
switch v := v.(*ModelOptions); i { |
||||
case 0: |
||||
return &v.state |
||||
case 1: |
||||
return &v.sizeCache |
||||
case 2: |
||||
return &v.unknownFields |
||||
default: |
||||
return nil |
||||
} |
||||
} |
||||
file_pkg_grpc_proto_llmserver_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { |
||||
switch v := v.(*Result); i { |
||||
case 0: |
||||
return &v.state |
||||
case 1: |
||||
return &v.sizeCache |
||||
case 2: |
||||
return &v.unknownFields |
||||
default: |
||||
return nil |
||||
} |
||||
} |
||||
file_pkg_grpc_proto_llmserver_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { |
||||
switch v := v.(*EmbeddingResult); i { |
||||
case 0: |
||||
return &v.state |
||||
case 1: |
||||
return &v.sizeCache |
||||
case 2: |
||||
return &v.unknownFields |
||||
default: |
||||
return nil |
||||
} |
||||
} |
||||
} |
||||
type x struct{} |
||||
out := protoimpl.TypeBuilder{ |
||||
File: protoimpl.DescBuilder{ |
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), |
||||
RawDescriptor: file_pkg_grpc_proto_llmserver_proto_rawDesc, |
||||
NumEnums: 0, |
||||
NumMessages: 6, |
||||
NumExtensions: 0, |
||||
NumServices: 1, |
||||
}, |
||||
GoTypes: file_pkg_grpc_proto_llmserver_proto_goTypes, |
||||
DependencyIndexes: file_pkg_grpc_proto_llmserver_proto_depIdxs, |
||||
MessageInfos: file_pkg_grpc_proto_llmserver_proto_msgTypes, |
||||
}.Build() |
||||
File_pkg_grpc_proto_llmserver_proto = out.File |
||||
file_pkg_grpc_proto_llmserver_proto_rawDesc = nil |
||||
file_pkg_grpc_proto_llmserver_proto_goTypes = nil |
||||
file_pkg_grpc_proto_llmserver_proto_depIdxs = nil |
||||
} |
||||
@ -1,277 +0,0 @@ |
||||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
// versions:
|
||||
// - protoc-gen-go-grpc v1.2.0
|
||||
// - protoc v3.15.8
|
||||
// source: pkg/grpc/proto/llmserver.proto
|
||||
|
||||
package proto |
||||
|
||||
import ( |
||||
context "context" |
||||
grpc "google.golang.org/grpc" |
||||
codes "google.golang.org/grpc/codes" |
||||
status "google.golang.org/grpc/status" |
||||
) |
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.32.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion7 |
||||
|
||||
// LLMClient is the client API for LLM service.
|
||||
//
|
||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||
type LLMClient interface { |
||||
Health(ctx context.Context, in *HealthMessage, opts ...grpc.CallOption) (*Reply, error) |
||||
Predict(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (*Reply, error) |
||||
LoadModel(ctx context.Context, in *ModelOptions, opts ...grpc.CallOption) (*Result, error) |
||||
PredictStream(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (LLM_PredictStreamClient, error) |
||||
Embedding(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (*EmbeddingResult, error) |
||||
} |
||||
|
||||
type lLMClient struct { |
||||
cc grpc.ClientConnInterface |
||||
} |
||||
|
||||
func NewLLMClient(cc grpc.ClientConnInterface) LLMClient { |
||||
return &lLMClient{cc} |
||||
} |
||||
|
||||
func (c *lLMClient) Health(ctx context.Context, in *HealthMessage, opts ...grpc.CallOption) (*Reply, error) { |
||||
out := new(Reply) |
||||
err := c.cc.Invoke(ctx, "/llm.LLM/Health", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *lLMClient) Predict(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (*Reply, error) { |
||||
out := new(Reply) |
||||
err := c.cc.Invoke(ctx, "/llm.LLM/Predict", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *lLMClient) LoadModel(ctx context.Context, in *ModelOptions, opts ...grpc.CallOption) (*Result, error) { |
||||
out := new(Result) |
||||
err := c.cc.Invoke(ctx, "/llm.LLM/LoadModel", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
func (c *lLMClient) PredictStream(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (LLM_PredictStreamClient, error) { |
||||
stream, err := c.cc.NewStream(ctx, &LLM_ServiceDesc.Streams[0], "/llm.LLM/PredictStream", opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
x := &lLMPredictStreamClient{stream} |
||||
if err := x.ClientStream.SendMsg(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if err := x.ClientStream.CloseSend(); err != nil { |
||||
return nil, err |
||||
} |
||||
return x, nil |
||||
} |
||||
|
||||
type LLM_PredictStreamClient interface { |
||||
Recv() (*Reply, error) |
||||
grpc.ClientStream |
||||
} |
||||
|
||||
type lLMPredictStreamClient struct { |
||||
grpc.ClientStream |
||||
} |
||||
|
||||
func (x *lLMPredictStreamClient) Recv() (*Reply, error) { |
||||
m := new(Reply) |
||||
if err := x.ClientStream.RecvMsg(m); err != nil { |
||||
return nil, err |
||||
} |
||||
return m, nil |
||||
} |
||||
|
||||
func (c *lLMClient) Embedding(ctx context.Context, in *PredictOptions, opts ...grpc.CallOption) (*EmbeddingResult, error) { |
||||
out := new(EmbeddingResult) |
||||
err := c.cc.Invoke(ctx, "/llm.LLM/Embedding", in, out, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
|
||||
// LLMServer is the server API for LLM service.
|
||||
// All implementations must embed UnimplementedLLMServer
|
||||
// for forward compatibility
|
||||
type LLMServer interface { |
||||
Health(context.Context, *HealthMessage) (*Reply, error) |
||||
Predict(context.Context, *PredictOptions) (*Reply, error) |
||||
LoadModel(context.Context, *ModelOptions) (*Result, error) |
||||
PredictStream(*PredictOptions, LLM_PredictStreamServer) error |
||||
Embedding(context.Context, *PredictOptions) (*EmbeddingResult, error) |
||||
mustEmbedUnimplementedLLMServer() |
||||
} |
||||
|
||||
// UnimplementedLLMServer must be embedded to have forward compatible implementations.
|
||||
type UnimplementedLLMServer struct { |
||||
} |
||||
|
||||
func (UnimplementedLLMServer) Health(context.Context, *HealthMessage) (*Reply, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method Health not implemented") |
||||
} |
||||
func (UnimplementedLLMServer) Predict(context.Context, *PredictOptions) (*Reply, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method Predict not implemented") |
||||
} |
||||
func (UnimplementedLLMServer) LoadModel(context.Context, *ModelOptions) (*Result, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method LoadModel not implemented") |
||||
} |
||||
func (UnimplementedLLMServer) PredictStream(*PredictOptions, LLM_PredictStreamServer) error { |
||||
return status.Errorf(codes.Unimplemented, "method PredictStream not implemented") |
||||
} |
||||
func (UnimplementedLLMServer) Embedding(context.Context, *PredictOptions) (*EmbeddingResult, error) { |
||||
return nil, status.Errorf(codes.Unimplemented, "method Embedding not implemented") |
||||
} |
||||
func (UnimplementedLLMServer) mustEmbedUnimplementedLLMServer() {} |
||||
|
||||
// UnsafeLLMServer may be embedded to opt out of forward compatibility for this service.
|
||||
// Use of this interface is not recommended, as added methods to LLMServer will
|
||||
// result in compilation errors.
|
||||
type UnsafeLLMServer interface { |
||||
mustEmbedUnimplementedLLMServer() |
||||
} |
||||
|
||||
func RegisterLLMServer(s grpc.ServiceRegistrar, srv LLMServer) { |
||||
s.RegisterService(&LLM_ServiceDesc, srv) |
||||
} |
||||
|
||||
func _LLM_Health_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(HealthMessage) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(LLMServer).Health(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/llm.LLM/Health", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(LLMServer).Health(ctx, req.(*HealthMessage)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _LLM_Predict_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(PredictOptions) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(LLMServer).Predict(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/llm.LLM/Predict", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(LLMServer).Predict(ctx, req.(*PredictOptions)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _LLM_LoadModel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(ModelOptions) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(LLMServer).LoadModel(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/llm.LLM/LoadModel", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(LLMServer).LoadModel(ctx, req.(*ModelOptions)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
func _LLM_PredictStream_Handler(srv interface{}, stream grpc.ServerStream) error { |
||||
m := new(PredictOptions) |
||||
if err := stream.RecvMsg(m); err != nil { |
||||
return err |
||||
} |
||||
return srv.(LLMServer).PredictStream(m, &lLMPredictStreamServer{stream}) |
||||
} |
||||
|
||||
type LLM_PredictStreamServer interface { |
||||
Send(*Reply) error |
||||
grpc.ServerStream |
||||
} |
||||
|
||||
type lLMPredictStreamServer struct { |
||||
grpc.ServerStream |
||||
} |
||||
|
||||
func (x *lLMPredictStreamServer) Send(m *Reply) error { |
||||
return x.ServerStream.SendMsg(m) |
||||
} |
||||
|
||||
func _LLM_Embedding_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(PredictOptions) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(LLMServer).Embedding(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/llm.LLM/Embedding", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(LLMServer).Embedding(ctx, req.(*PredictOptions)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
|
||||
// LLM_ServiceDesc is the grpc.ServiceDesc for LLM service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
var LLM_ServiceDesc = grpc.ServiceDesc{ |
||||
ServiceName: "llm.LLM", |
||||
HandlerType: (*LLMServer)(nil), |
||||
Methods: []grpc.MethodDesc{ |
||||
{ |
||||
MethodName: "Health", |
||||
Handler: _LLM_Health_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "Predict", |
||||
Handler: _LLM_Predict_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "LoadModel", |
||||
Handler: _LLM_LoadModel_Handler, |
||||
}, |
||||
{ |
||||
MethodName: "Embedding", |
||||
Handler: _LLM_Embedding_Handler, |
||||
}, |
||||
}, |
||||
Streams: []grpc.StreamDesc{ |
||||
{ |
||||
StreamName: "PredictStream", |
||||
Handler: _LLM_PredictStream_Handler, |
||||
ServerStreams: true, |
||||
}, |
||||
}, |
||||
Metadata: "pkg/grpc/proto/llmserver.proto", |
||||
} |
||||
@ -0,0 +1,27 @@ |
||||
package transcribe |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper" |
||||
"github.com/go-skynet/LocalAI/pkg/grpc/base" |
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
whisperutil "github.com/go-skynet/LocalAI/pkg/grpc/whisper" |
||||
"github.com/go-skynet/LocalAI/pkg/grpc/whisper/api" |
||||
) |
||||
|
||||
type Whisper struct { |
||||
base.Base |
||||
whisper whisper.Model |
||||
} |
||||
|
||||
func (sd *Whisper) Load(opts *pb.ModelOptions) error { |
||||
// Note: the Model here is a path to a directory containing the model files
|
||||
w, err := whisper.New(opts.Model) |
||||
sd.whisper = w |
||||
return err |
||||
} |
||||
|
||||
func (sd *Whisper) AudioTranscription(opts *pb.TranscriptRequest) (api.Result, error) { |
||||
return whisperutil.Transcript(sd.whisper, opts.Dst, opts.Language, uint(opts.Threads)) |
||||
} |
||||
@ -0,0 +1,44 @@ |
||||
package tts |
||||
|
||||
// This is a wrapper to statisfy the GRPC service interface
|
||||
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||||
import ( |
||||
"os" |
||||
|
||||
"github.com/go-skynet/LocalAI/pkg/grpc/base" |
||||
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto" |
||||
piper "github.com/mudler/go-piper" |
||||
) |
||||
|
||||
type Piper struct { |
||||
base.Base |
||||
piper *PiperB |
||||
} |
||||
|
||||
func (sd *Piper) Load(opts *pb.ModelOptions) error { |
||||
var err error |
||||
// Note: the Model here is a path to a directory containing the model files
|
||||
sd.piper, err = New(opts.LibrarySearchPath) |
||||
return err |
||||
} |
||||
|
||||
func (sd *Piper) TTS(opts *pb.TTSRequest) error { |
||||
return sd.piper.TTS(opts.Text, opts.Model, opts.Dst) |
||||
} |
||||
|
||||
type PiperB struct { |
||||
assetDir string |
||||
} |
||||
|
||||
func New(assetDir string) (*PiperB, error) { |
||||
if _, err := os.Stat(assetDir); err != nil { |
||||
return nil, err |
||||
} |
||||
return &PiperB{ |
||||
assetDir: assetDir, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *PiperB) TTS(text, model, dst string) error { |
||||
return piper.TextToWav(text, model, s.assetDir, "", dst) |
||||
} |
||||
@ -0,0 +1,16 @@ |
||||
package api |
||||
|
||||
import "time" |
||||
|
||||
type Segment struct { |
||||
Id int `json:"id"` |
||||
Start time.Duration `json:"start"` |
||||
End time.Duration `json:"end"` |
||||
Text string `json:"text"` |
||||
Tokens []int `json:"tokens"` |
||||
} |
||||
|
||||
type Result struct { |
||||
Segments []Segment `json:"segments"` |
||||
Text string `json:"text"` |
||||
} |
||||
@ -1,12 +0,0 @@ |
||||
//go:build tts
|
||||
// +build tts
|
||||
|
||||
package tts |
||||
|
||||
import ( |
||||
piper "github.com/mudler/go-piper" |
||||
) |
||||
|
||||
func tts(text, model, assetDir, arLib, dst string) error { |
||||
return piper.TextToWav(text, model, assetDir, arLib, dst) |
||||
} |
||||
@ -1,10 +0,0 @@ |
||||
//go:build !tts
|
||||
// +build !tts
|
||||
|
||||
package tts |
||||
|
||||
import "fmt" |
||||
|
||||
func tts(text, model, assetDir, arLib, dst string) error { |
||||
return fmt.Errorf("this version of LocalAI was built without the tts tag") |
||||
} |
||||
@ -1,20 +0,0 @@ |
||||
package tts |
||||
|
||||
import "os" |
||||
|
||||
type Piper struct { |
||||
assetDir string |
||||
} |
||||
|
||||
func New(assetDir string) (*Piper, error) { |
||||
if _, err := os.Stat(assetDir); err != nil { |
||||
return nil, err |
||||
} |
||||
return &Piper{ |
||||
assetDir: assetDir, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *Piper) TTS(text, model, dst string) error { |
||||
return tts(text, model, s.assetDir, "", dst) |
||||
} |
||||
Loading…
Reference in new issue