diff --git a/go/cmd/go.mod b/go/cmd/go.mod new file mode 100644 index 00000000..7e795dcd --- /dev/null +++ b/go/cmd/go.mod @@ -0,0 +1,9 @@ +module main + +go 1.22.4 + +replace github.com/ling0322/libllm/go/llm => ../llm + +require ( + github.com/ling0322/libllm/go/llm v1.0.0 +) diff --git a/go/cmd/main.go b/go/cmd/main.go new file mode 100644 index 00000000..4183e4b8 --- /dev/null +++ b/go/cmd/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "fmt" + "log" + + "github.com/ling0322/libllm/go/llm" +) + +func main() { + model, err := llm.NewModel("../../tools/llama.llmpkg", llm.Cuda) + if err != nil { + log.Fatal(err) + } + + prompt := llm.NewPrompt() + prompt.AppendControlToken("<|begin_of_text|>") + prompt.AppendControlToken("<|start_header_id|>") + prompt.AppendText("user") + prompt.AppendControlToken("<|end_header_id|>") + prompt.AppendText("\n\nHi") + prompt.AppendControlToken("<|eot_id|>") + prompt.AppendControlToken("<|start_header_id|>") + prompt.AppendText("assistant") + prompt.AppendControlToken("<|end_header_id|>") + prompt.AppendText("\n\n") + + log.Println(model) + + comp, err := model.Complete(llm.NewCompletionConfig(), prompt) + if err != nil { + log.Fatal(err) + } + + for comp.IsActive() { + chunk, err := comp.GenerateNextChunk() + if err != nil { + log.Fatal(err) + } + fmt.Printf(chunk.Text) + } +} diff --git a/bindings/go/libllm/llm.go b/go/llm/chunk.go similarity index 62% rename from bindings/go/libllm/llm.go rename to go/llm/chunk.go index 95154f7c..52fac61e 100644 --- a/bindings/go/libllm/llm.go +++ b/go/llm/chunk.go @@ -1,6 +1,6 @@ // The MIT License (MIT) // -// Copyright (c) 2023 Xiaoyang Chen +// Copyright (c) 2024 Xiaoyang Chen // // Permission is hereby granted, free of charge, to any person obtaining a copy of this software // and associated documentation files (the "Software"), to deal in the Software without @@ -17,26 +17,42 @@ // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -package libllm +package llm +// #include +// #include "llm_api.h" +import "C" import ( - "unsafe" + "errors" + "fmt" + "os" + "runtime" ) -type Device int - -const ( - Cpu Device = iota - Cuda - Auto -) - -type Model struct { - handle unsafe.Pointer +// Generate by Compeltion. +type Chunk struct { + Text string } -func LoadModel(filename string) (model *Model, err error) { - +type chunkHandle struct { + handle *C.llmChunk_t } -func (m *Model) Complete() +func newChunkHandle() (*chunkHandle, error) { + cHandle := C.llmChunk_New() + if cHandle == nil { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + handle := &chunkHandle{ + cHandle, + } + runtime.SetFinalizer(handle, func(h *chunkHandle) { + status := C.llmChunk_Delete(h.handle) + if status != C.LLM_OK { + fmt.Fprintln(os.Stderr, "failed to call llmPrompt_Delete()") + } + }) + + return handle, nil +} diff --git a/go/llm/completion.go b/go/llm/completion.go new file mode 100644 index 00000000..f6e220de --- /dev/null +++ b/go/llm/completion.go @@ -0,0 +1,101 @@ +// The MIT License (MIT) +// +// Copyright (c) 2024 Xiaoyang Chen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software +// and associated documentation files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or +// substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package llm + +// #include +// #include "llm_api.h" +import "C" +import ( + "errors" + "fmt" + "os" + "runtime" +) + +// Config for LLM completion. +type Completion interface { + IsActive() bool + GenerateNextChunk() (Chunk, error) +} + +type completionHandle struct { + handle *C.llmCompletion_t +} + +type completionImpl struct { + handle *completionHandle + chunkHandle *chunkHandle +} + +func (c *completionImpl) IsActive() bool { + return C.llmCompletion_IsActive(c.handle.handle) != 0 +} + +func (c *completionImpl) GenerateNextChunk() (Chunk, error) { + status := C.llmCompletion_GenerateNextChunk(c.handle.handle, c.chunkHandle.handle) + if status != C.LLM_OK { + return Chunk{}, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + chunk := Chunk{} + cText := C.llmChunk_GetText(c.chunkHandle.handle) + if cText == nil { + return Chunk{}, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + chunk.Text = C.GoString(cText) + return chunk, nil +} + +func newCompletionImpl(modelHandle *modelHandle) (*completionImpl, error) { + handle, err := newCompletionHandle(modelHandle) + if err != nil { + return nil, err + } + + chunkHandle, err := newChunkHandle() + if err != nil { + return nil, err + } + + return &completionImpl{ + handle: handle, + chunkHandle: chunkHandle, + }, nil +} + +func newCompletionHandle(m *modelHandle) (*completionHandle, error) { + cHandle := C.llmCompletion_New(m.handle) + if cHandle == nil { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + handle := &completionHandle{ + cHandle, + } + runtime.SetFinalizer(handle, func(h *completionHandle) { + status := C.llmCompletion_Delete(h.handle) + if status != C.LLM_OK { + fmt.Fprintln(os.Stderr, "failed to call llmCompletion_Delete()") + } + }) + + return handle, nil +} diff --git a/go/llm/completion_config.go b/go/llm/completion_config.go new file mode 100644 index 00000000..dbaecfff --- /dev/null +++ b/go/llm/completion_config.go @@ -0,0 +1,94 @@ +// The MIT License (MIT) +// +// Copyright (c) 2024 Xiaoyang Chen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software +// and associated documentation files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or +// substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package llm + +// #include +// #include "llm_api.h" +import "C" +import "errors" + +// Config for LLM completion. +type CompletionConfig interface { + SetTopP(topP float32) + GetTopP() float32 + + SetTopK(topK int) + GetTopK() int + + SetTemperature(temperature float32) + GetTemperature() float32 + + // update the llmCompletion_t according to the config. + updateCompHandle(compHandle *completionHandle) error +} + +type completionConfigImpl struct { + topP float32 + topK int + temperature float32 +} + +func NewCompletionConfig() CompletionConfig { + return &completionConfigImpl{ + topP: 0.8, + topK: 50, + temperature: 1.0, + } +} + +func (c *completionConfigImpl) SetTopP(topP float32) { + c.topP = topP +} + +func (c *completionConfigImpl) GetTopP() float32 { + return c.topP +} + +func (c *completionConfigImpl) SetTopK(topK int) { + c.topK = topK +} + +func (c *completionConfigImpl) GetTopK() int { + return c.topK +} + +func (c *completionConfigImpl) SetTemperature(temperature float32) { + c.temperature = temperature +} + +func (c *completionConfigImpl) GetTemperature() float32 { + return c.temperature +} + +func (c *completionConfigImpl) updateCompHandle(compHandle *completionHandle) error { + if C.llmCompletion_SetTopP(compHandle.handle, C.float(c.topP)) != C.LLM_OK { + return errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + if C.llmCompletion_SetTopK(compHandle.handle, C.int(c.topK)) != C.LLM_OK { + return errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + if C.llmCompletion_SetTemperature(compHandle.handle, C.float(c.temperature)) != C.LLM_OK { + return errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + return nil +} diff --git a/go/llm/go.mod b/go/llm/go.mod new file mode 100644 index 00000000..708f9126 --- /dev/null +++ b/go/llm/go.mod @@ -0,0 +1,3 @@ +module github.com/ling0322/libllm/go/llm + +go 1.22.4 diff --git a/go/llm/llm.go b/go/llm/llm.go new file mode 100644 index 00000000..44010fdf --- /dev/null +++ b/go/llm/llm.go @@ -0,0 +1,108 @@ +// The MIT License (MIT) +// +// Copyright (c) 2024 Xiaoyang Chen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software +// and associated documentation files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or +// substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package llm + +// #cgo LDFLAGS: -ldl +// #include +// #include "llm_api.h" +import "C" +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "unsafe" +) + +type Device int32 + +const ( + Cpu = Device(0x0000) + Cuda = Device(0x0100) + Auto = Device(0x1f00) +) + +var gInit atomic.Bool +var gDll unsafe.Pointer + +// Initialize the libllm. +func initLlm() error { + if !gInit.Swap(true) { + // load the shared library. + binPath, err := os.Executable() + if err != nil { + gInit.Store(false) + return err + } + + binDir := filepath.Dir(binPath) + dllPath := C.CString(filepath.Join(binDir, "libllm.so")) + defer C.free(unsafe.Pointer(dllPath)) + + gDll = C.llmLoadLibrary(dllPath) + if gDll == nil { + dllPath := C.CString("libllm.so") + defer C.free(unsafe.Pointer(dllPath)) + + gDll = C.llmLoadLibrary(dllPath) + } + + if gDll == nil { + gInit.Store(false) + return errors.New("failed to load the libllm dynamic library") + } + + // initialize the symbols. + status := C.llmLoadSymbols(gDll) + if status != C.LLM_OK { + gInit.Store(false) + return errors.New("failed to load libllm api symbols") + } + + // initialize libllm inference engine. + status = C.llmInit(C.LLM_API_VERSION) + if status != C.LLM_OK { + gInit.Store(false) + return errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + } + + return nil +} + +// Release all the resources allocated in libllm library. +func Release() { + if gInit.Swap(false) { + status := C.llmDestroy() + if status != C.LLM_OK { + fmt.Fprintf( + os.Stderr, + "failed to destroy libllm: %s\n", + C.GoString(C.llmGetLastErrorMessage())) + } + + // release the dynamic library itself. + status = C.llmDestroyLibrary(gDll) + if status != C.LLM_OK { + fmt.Fprintf(os.Stderr, "failed to close dynamic library of libllm\n") + } + } +} diff --git a/go/llm/llm_api.c b/go/llm/llm_api.c new file mode 100644 index 00000000..ff379b8a --- /dev/null +++ b/go/llm/llm_api.c @@ -0,0 +1,261 @@ +// The MIT License (MIT) +// +// Copyright (c) 2024 Xiaoyang Chen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software +// and associated documentation files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or +// substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#ifndef LIBLLM_LLM_API_ +#define LIBLLM_LLM_API_ + +#include +#include +#include +#include + +#define LLM_DEVICE_CPU 0x0000 +#define LLM_DEVICE_CUDA 0x0100 +#define LLM_DEVICE_AUTO 0x1f00 +#define LLM_API_VERSION 20240101 +#define LLM_OK 0 +#define LLM_ABORTED 1 + +typedef int32_t llmStatus_t; +typedef struct llmModel_t llmModel_t; +typedef struct llmChunk_t llmChunk_t; +typedef struct llmPrompt_t llmPrompt_t; +typedef struct llmCompletion_t llmCompletion_t; +typedef int32_t llmBool_t; + +// global state +llmStatus_t (*p_llmInit)(int32_t apiVersion); +llmStatus_t (*p_llmDestroy)(); +const char *(*p_llmGetLastErrorMessage)(); + +// llmModel_t +llmModel_t *(*p_llmModel_New)(); +llmStatus_t (*p_llmModel_Delete)(llmModel_t *model); +llmStatus_t (*p_llmModel_SetFile)(llmModel_t *model, const char *filename); +llmStatus_t (*p_llmModel_SetDevice)(llmModel_t *model, int32_t device); +llmStatus_t (*p_llmModel_Load)(llmModel_t *model); +const char *(*p_llmModel_GetName)(llmModel_t *model); + +// llmPrompt_t +llmPrompt_t *(*p_llmPrompt_New)(llmModel_t *model); +llmStatus_t (*p_llmPrompt_Delete)(llmPrompt_t *prompt); +llmStatus_t (*p_llmPrompt_AppendText)(llmPrompt_t *prompt, const char *text); +llmStatus_t (*p_llmPrompt_AppendControlToken)(llmPrompt_t *prompt, const char *token); + +// llmCompletion_t +llmCompletion_t *(*p_llmCompletion_New)(llmModel_t *model); +llmStatus_t (*p_llmCompletion_Delete)(llmCompletion_t *comp); +llmStatus_t (*p_llmCompletion_SetPrompt)(llmCompletion_t *comp, llmPrompt_t *prompt); +llmStatus_t (*p_llmCompletion_SetTopP)(llmCompletion_t *comp, float topP); +llmStatus_t (*p_llmCompletion_SetTopK)(llmCompletion_t *comp, int32_t topK); +llmStatus_t (*p_llmCompletion_SetTemperature)(llmCompletion_t *comp, float temperature); +llmStatus_t (*p_llmCompletion_Start)(llmCompletion_t *comp); +llmBool_t (*p_llmCompletion_IsActive)(llmCompletion_t *comp); +llmStatus_t (*p_llmCompletion_GenerateNextChunk)(llmCompletion_t *comp, llmChunk_t *chunk); + +// llmChunk_t +llmChunk_t *(*p_llmChunk_New)(); +llmStatus_t (*p_llmChunk_Delete)(llmChunk_t *chunk); +const char *(*p_llmChunk_GetText)(llmChunk_t *chunk); + +// load the libllm shared library. +void *llmLoadLibrary(const char *libraryPath) { + // first try to load the dll from same folder as current module. + return dlopen(libraryPath, RTLD_NOW); +} + +#define LOAD_SYMBOL(hDll, symbol) \ + p_##symbol = dlsym(hDll, #symbol); \ + if (!p_##symbol) { \ + fprintf(stderr, "llm.go: unable to load symbol: %s\n", #symbol); \ + return LLM_ABORTED; \ + } + +llmStatus_t llmLoadSymbols(void *hDll) { + LOAD_SYMBOL(hDll, llmInit); + LOAD_SYMBOL(hDll, llmDestroy); + LOAD_SYMBOL(hDll, llmGetLastErrorMessage); + LOAD_SYMBOL(hDll, llmModel_New); + LOAD_SYMBOL(hDll, llmModel_Delete); + LOAD_SYMBOL(hDll, llmModel_SetFile); + LOAD_SYMBOL(hDll, llmModel_SetDevice); + LOAD_SYMBOL(hDll, llmModel_Load); + LOAD_SYMBOL(hDll, llmModel_GetName); + LOAD_SYMBOL(hDll, llmPrompt_New); + LOAD_SYMBOL(hDll, llmPrompt_Delete); + LOAD_SYMBOL(hDll, llmPrompt_AppendText); + LOAD_SYMBOL(hDll, llmPrompt_AppendControlToken); + LOAD_SYMBOL(hDll, llmCompletion_New); + LOAD_SYMBOL(hDll, llmCompletion_Delete); + LOAD_SYMBOL(hDll, llmCompletion_SetPrompt); + LOAD_SYMBOL(hDll, llmCompletion_SetTopP); + LOAD_SYMBOL(hDll, llmCompletion_SetTopK); + LOAD_SYMBOL(hDll, llmCompletion_SetTemperature); + LOAD_SYMBOL(hDll, llmCompletion_Start); + LOAD_SYMBOL(hDll, llmCompletion_IsActive); + LOAD_SYMBOL(hDll, llmCompletion_GenerateNextChunk); + LOAD_SYMBOL(hDll, llmChunk_New); + LOAD_SYMBOL(hDll, llmChunk_Delete); + LOAD_SYMBOL(hDll, llmChunk_GetText); + + return LLM_OK; +} + +// load the libllm shared library. +llmStatus_t llmDestroyLibrary(void *handle) { + p_llmInit = NULL; + p_llmDestroy = NULL; + p_llmGetLastErrorMessage = NULL; + p_llmModel_New = NULL; + p_llmModel_Delete = NULL; + p_llmModel_SetFile = NULL; + p_llmModel_SetDevice = NULL; + p_llmModel_Load = NULL; + p_llmModel_GetName = NULL; + p_llmPrompt_New = NULL; + p_llmPrompt_Delete = NULL; + p_llmPrompt_AppendText = NULL; + p_llmPrompt_AppendControlToken = NULL; + p_llmCompletion_New = NULL; + p_llmCompletion_Delete = NULL; + p_llmCompletion_SetPrompt = NULL; + p_llmCompletion_SetTopP = NULL; + p_llmCompletion_SetTopK = NULL; + p_llmCompletion_SetTemperature = NULL; + p_llmCompletion_Start = NULL; + p_llmCompletion_IsActive = NULL; + p_llmCompletion_GenerateNextChunk = NULL; + p_llmChunk_New = NULL; + p_llmChunk_Delete = NULL; + p_llmChunk_GetText = NULL; + + // first try to load the dll from same folder as current module. + int ret = dlclose(handle); + if (ret != 0) { + return LLM_ABORTED; + } + + return LLM_OK; +} + +llmStatus_t llmInit(int32_t apiVersion) { + return p_llmInit(apiVersion); +} + +llmStatus_t llmDestroy() { + return p_llmDestroy(); +} + +const char *llmGetLastErrorMessage() { + return p_llmGetLastErrorMessage(); +} + +// llmModel_t +llmModel_t *llmModel_New() { + return p_llmModel_New(); +} + +llmStatus_t llmModel_Delete(llmModel_t *model) { + return p_llmModel_Delete(model); +} + +llmStatus_t llmModel_SetFile(llmModel_t *model, const char *filename) { + return p_llmModel_SetFile(model, filename); +} + +llmStatus_t llmModel_SetDevice(llmModel_t *model, int32_t device) { + return p_llmModel_SetDevice(model, device); +} + +llmStatus_t llmModel_Load(llmModel_t *model) { + return p_llmModel_Load(model); +} + +const char *llmModel_GetName(llmModel_t *model) { + return p_llmModel_GetName(model); +} + +// llmPrompt_t +llmPrompt_t *llmPrompt_New(llmModel_t *model) { + return p_llmPrompt_New(model); +} + +llmStatus_t llmPrompt_Delete(llmPrompt_t *prompt) { + return p_llmPrompt_Delete(prompt); +} + +llmStatus_t llmPrompt_AppendText(llmPrompt_t *prompt, const char *text) { + return p_llmPrompt_AppendText(prompt, text); +} + +llmStatus_t llmPrompt_AppendControlToken(llmPrompt_t *prompt, const char *token) { + return p_llmPrompt_AppendControlToken(prompt, token); +} + +// llmCompletion_t +llmCompletion_t *llmCompletion_New(llmModel_t *model) { + return p_llmCompletion_New(model); +} + +llmStatus_t llmCompletion_Delete(llmCompletion_t *comp) { + return p_llmCompletion_Delete(comp); +} + +llmStatus_t llmCompletion_SetPrompt(llmCompletion_t *comp, llmPrompt_t *prompt) { + return p_llmCompletion_SetPrompt(comp, prompt); +} + +llmStatus_t llmCompletion_SetTopP(llmCompletion_t *comp, float topP) { + return p_llmCompletion_SetTopP(comp, topP); +} + +llmStatus_t llmCompletion_SetTopK(llmCompletion_t *comp, int32_t topK) { + return p_llmCompletion_SetTopK(comp, topK); +} + +llmStatus_t llmCompletion_SetTemperature(llmCompletion_t *comp, float temperature) { + return p_llmCompletion_SetTemperature(comp, temperature); +} + +llmStatus_t llmCompletion_Start(llmCompletion_t *comp) { + return p_llmCompletion_Start(comp); +} + +llmBool_t llmCompletion_IsActive(llmCompletion_t *comp) { + return p_llmCompletion_IsActive(comp); +} + +llmStatus_t llmCompletion_GenerateNextChunk(llmCompletion_t *comp, llmChunk_t *chunk) { + return p_llmCompletion_GenerateNextChunk(comp, chunk); +} + +// llmChunk_t +llmChunk_t *llmChunk_New() { + return p_llmChunk_New(); +} + +llmStatus_t llmChunk_Delete(llmChunk_t *chunk) { + return p_llmChunk_Delete(chunk); +} + +const char *llmChunk_GetText(llmChunk_t *chunk) { + return p_llmChunk_GetText(chunk); +} + +#endif // LIBLLM_LLM_API_ diff --git a/go/llm/llm_api.h b/go/llm/llm_api.h new file mode 100644 index 00000000..dd1971e5 --- /dev/null +++ b/go/llm/llm_api.h @@ -0,0 +1,77 @@ +// The MIT License (MIT) +// +// Copyright (c) 2023 Xiaoyang Chen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software +// and associated documentation files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or +// substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#ifndef LIBLLM_LLM_API_ +#define LIBLLM_LLM_API_ + +#include + +#define LLM_DEVICE_CPU 0x0000 +#define LLM_DEVICE_CUDA 0x0100 +#define LLM_DEVICE_AUTO 0x1f00 +#define LLM_API_VERSION 20240101 +#define LLM_OK 0 + +typedef int32_t llmStatus_t; +typedef struct llmModel_t llmModel_t; +typedef struct llmChunk_t llmChunk_t; +typedef struct llmPrompt_t llmPrompt_t; +typedef struct llmCompletion_t llmCompletion_t; +typedef int32_t llmBool_t; + +void *llmLoadLibrary(const char *libraryPath); +llmStatus_t llmLoadSymbols(void *hDll); +llmStatus_t llmDestroyLibrary(void *handle); + +// global state +llmStatus_t llmInit(int32_t apiVersion); +llmStatus_t llmDestroy(); +const char *llmGetLastErrorMessage(); + +// llmModel_t +llmModel_t *llmModel_New(); +llmStatus_t llmModel_Delete(llmModel_t *model); +llmStatus_t llmModel_SetFile(llmModel_t *model, const char *filename); +llmStatus_t llmModel_SetDevice(llmModel_t *model, int32_t device); +llmStatus_t llmModel_Load(llmModel_t *model); +const char *llmModel_GetName(llmModel_t *model); + +// llmPrompt_t +llmPrompt_t *llmPrompt_New(llmModel_t *model); +llmStatus_t llmPrompt_Delete(llmPrompt_t *prompt); +llmStatus_t llmPrompt_AppendText(llmPrompt_t *prompt, const char *text); +llmStatus_t llmPrompt_AppendControlToken(llmPrompt_t *prompt, const char *token); + +// llmCompletion_t +llmCompletion_t *llmCompletion_New(llmModel_t *model); +llmStatus_t llmCompletion_Delete(llmCompletion_t *comp); +llmStatus_t llmCompletion_SetPrompt(llmCompletion_t *comp, llmPrompt_t *prompt); +llmStatus_t llmCompletion_SetTopP(llmCompletion_t *comp, float topP); +llmStatus_t llmCompletion_SetTopK(llmCompletion_t *comp, int32_t topK); +llmStatus_t llmCompletion_SetTemperature(llmCompletion_t *comp, float temperature); +llmStatus_t llmCompletion_Start(llmCompletion_t *comp); +llmBool_t llmCompletion_IsActive(llmCompletion_t *comp); +llmStatus_t llmCompletion_GenerateNextChunk(llmCompletion_t *comp, llmChunk_t *chunk); + +// llmChunk_t +llmChunk_t *llmChunk_New(); +llmStatus_t llmChunk_Delete(llmChunk_t *chunk); +const char *llmChunk_GetText(llmChunk_t *chunk); + +#endif // LIBLLM_LLM_API_ diff --git a/go/llm/model.go b/go/llm/model.go new file mode 100644 index 00000000..c3c4dea2 --- /dev/null +++ b/go/llm/model.go @@ -0,0 +1,140 @@ +// The MIT License (MIT) +// +// Copyright (c) 2024 Xiaoyang Chen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software +// and associated documentation files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or +// substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package llm + +// #include +// #include "llm_api.h" +import "C" +import ( + "errors" + "fmt" + "os" + "runtime" + "unsafe" +) + +// A LLM. +type Model interface { + GetName() string + Complete(config CompletionConfig, prompt Prompt) (Completion, error) +} + +type modelHandle struct { + handle *C.llmModel_t +} + +type modelImpl struct { + handle *modelHandle +} + +// Load a LLM model from `modelPath`, then save it to the specified device. +func NewModel(modelPath string, device Device) (Model, error) { + err := initLlm() + if err != nil { + return nil, err + } + + handle, err := newModelHandle() + if err != nil { + return nil, err + } + + cPath := C.CString(modelPath) + defer C.free(unsafe.Pointer(cPath)) + if C.llmModel_SetFile(handle.handle, cPath) != C.LLM_OK { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + if C.llmModel_SetDevice(handle.handle, C.int32_t(device)) != C.LLM_OK { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + if C.llmModel_Load(handle.handle) != C.LLM_OK { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + model := &modelImpl{ + handle: handle, + } + return model, nil +} + +func (m *modelImpl) Complete(config CompletionConfig, prompt Prompt) (Completion, error) { + comp, err := newCompletionImpl(m.handle) + if err != nil { + return nil, err + } + + err = config.updateCompHandle(comp.handle) + if err != nil { + return nil, err + } + + promptHandle, err := newPromptHandle(m.handle) + if err != nil { + return nil, err + } + + err = prompt.updatePromptHandle(promptHandle) + if err != nil { + return nil, err + } + + ok := C.llmCompletion_SetPrompt(comp.handle.handle, promptHandle.handle) + if ok != C.LLM_OK { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + ok = C.llmCompletion_Start(comp.handle.handle) + if ok != C.LLM_OK { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + return comp, nil +} + +// Get the name of model. +func (m *modelImpl) GetName() string { + name := C.llmModel_GetName(m.handle.handle) + if name == nil { + return "" + } else { + return C.GoString(name) + } +} + +func newModelHandle() (*modelHandle, error) { + cHandle := C.llmModel_New() + if cHandle == nil { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + handle := &modelHandle{ + cHandle, + } + runtime.SetFinalizer(handle, func(h *modelHandle) { + status := C.llmModel_Delete(h.handle) + if status != C.LLM_OK { + fmt.Fprintln(os.Stderr, "failed to call llmModel_Delete()") + } + }) + + return handle, nil +} diff --git a/go/llm/prompt.go b/go/llm/prompt.go new file mode 100644 index 00000000..d318b3e2 --- /dev/null +++ b/go/llm/prompt.go @@ -0,0 +1,130 @@ +// The MIT License (MIT) +// +// Copyright (c) 2024 Xiaoyang Chen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software +// and associated documentation files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or +// substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package llm + +// #include +// #include "llm_api.h" +import "C" +import ( + "errors" + "fmt" + "os" + "runtime" + "unsafe" +) + +// The input of LLM. +type Prompt interface { + AppendText(text string) + AppendControlToken(text string) + + // Update the llmPrompt_t instance according to the current prompt. + updatePromptHandle(handle *promptHandle) error +} + +type promptImpl struct { + elements []promptElem +} + +type promptElem interface { + AppendTo(handle *promptHandle) error +} + +type textPromptElem struct { + text string +} + +type controlTokenPromptElem struct { + token string +} + +type promptHandle struct { + handle *C.llmPrompt_t +} + +func NewPrompt() Prompt { + return &promptImpl{} +} + +func (p *promptImpl) AppendText(text string) { + p.elements = append(p.elements, &textPromptElem{text}) +} + +func (p *promptImpl) AppendControlToken(text string) { + p.elements = append(p.elements, &textPromptElem{text}) +} + +func (p *promptImpl) updatePromptHandle(handle *promptHandle) error { + if len(p.elements) == 0 { + return errors.New("prompt is empty") + } + + for _, elem := range p.elements { + err := elem.AppendTo(handle) + if err != nil { + return err + } + } + + return nil +} + +func (e *textPromptElem) AppendTo(handle *promptHandle) error { + cText := C.CString(e.text) + defer C.free(unsafe.Pointer(cText)) + + status := C.llmPrompt_AppendText(handle.handle, cText) + if status != C.LLM_OK { + return errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + return nil +} + +func (e *controlTokenPromptElem) AppendTo(handle *promptHandle) error { + cToken := C.CString(e.token) + defer C.free(unsafe.Pointer(cToken)) + + status := C.llmPrompt_AppendControlToken(handle.handle, cToken) + if status != C.LLM_OK { + return errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + return nil +} + +func newPromptHandle(m *modelHandle) (*promptHandle, error) { + cHandle := C.llmPrompt_New(m.handle) + if cHandle == nil { + return nil, errors.New(C.GoString(C.llmGetLastErrorMessage())) + } + + handle := &promptHandle{ + cHandle, + } + runtime.SetFinalizer(handle, func(h *promptHandle) { + status := C.llmPrompt_Delete(h.handle) + if status != C.LLM_OK { + fmt.Fprintln(os.Stderr, "failed to call llmPrompt_Delete()") + } + }) + + return handle, nil +}