You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
43 lines
1.1 KiB
43 lines
1.1 KiB
1 year ago
|
package transformers
|
||
|
|
|
||
|
|
// This is a wrapper to statisfy the GRPC service interface
|
||
|
|
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
||
|
|
import (
|
||
|
|
"fmt"
|
||
|
|
|
||
|
|
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
|
||
|
|
|
||
|
|
transformers "github.com/go-skynet/go-ggml-transformers.cpp"
|
||
|
|
)
|
||
|
|
|
||
|
|
type Starcoder struct {
|
||
|
|
starcoder *transformers.Starcoder
|
||
|
|
}
|
||
|
|
|
||
|
|
func (llm *Starcoder) Load(opts *pb.ModelOptions) error {
|
||
|
|
model, err := transformers.NewStarcoder(opts.Model)
|
||
|
|
llm.starcoder = model
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
func (llm *Starcoder) Embeddings(opts *pb.PredictOptions) ([]float32, error) {
|
||
|
|
return nil, fmt.Errorf("not implemented")
|
||
|
|
}
|
||
|
|
|
||
|
|
func (llm *Starcoder) Predict(opts *pb.PredictOptions) (string, error) {
|
||
|
|
return llm.starcoder.Predict(opts.Prompt, buildPredictOptions(opts)...)
|
||
|
|
}
|
||
|
|
|
||
|
|
// fallback to Predict
|
||
|
|
func (llm *Starcoder) PredictStream(opts *pb.PredictOptions, results chan string) {
|
||
|
|
go func() {
|
||
|
|
res, err := llm.starcoder.Predict(opts.Prompt, buildPredictOptions(opts)...)
|
||
|
|
|
||
|
|
if err != nil {
|
||
|
|
fmt.Println("err: ", err)
|
||
|
|
}
|
||
|
|
results <- res
|
||
|
|
close(results)
|
||
|
|
}()
|
||
|
|
}
|