From 5db853767b4f14e8eea1304a278b6602e4f12ac4 Mon Sep 17 00:00:00 2001 From: Muhammad Azeez Date: Tue, 19 Sep 2023 15:37:23 +0300 Subject: [PATCH 1/5] docs: update README to contain some basic usage examples --- README.md | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 181 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5549c75..7886daa 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,190 @@ # Extism Go SDK -For more information about the Go SDK, please visit our docs. +> **Note**: This houses the 1.0 version of the Go SDK and is a work in progress. Please use the Go SDK in [extism/extism](https://github.com/extism/extism) until we hit 1.0. + +This repo houses the Go SDK for integrating with the [Extism](https://extism.org/) runtime. Install this library into your host Go applications to run Extism plugins. Join the [Discord](https://discord.gg/EGTV8Pxs) and chat with us! -## Build example plugins +## Installation + +Install via `go get`: + +``` +go get github.com/extism/go-sdk +``` + +## Getting Started + +The primary concept in Extism is the plug-in. You can think of a plug-in as a code module. It has imports and it has exports. These imports and exports define the interface, or your API. You decide what they are called and typed, and what they do. Then the plug-in developer implements them and you can call them. + +The code for a plug-in exist as a binary wasm module. We can load this with the raw bytes or we can use the manifest to tell Extism how to load it from disk or the web. + +For simplicity let's load one from the web: + +```go +// NOTE: The schema for this manifest can be found here: https://extism.org/docs/concepts/manifest/ +manifest := extism.Manifest{ + Wasm: []extism.Wasm{ + extism.WasmUrl{ + Url: "https://raw.githubusercontent.com/extism/extism/main/wasm/code.wasm", + }, + }, +} + +ctx := context.Background() +config := extism.PluginConfig{ + EnableWasi: true, +} + +// NOTE: if you encounter an error such as: +// "Unable to load plugin: unknown import: wasi_snapshot_preview1::fd_write has not been defined" +// make sure extism.PluginConfig.EnableWasi is set to `true` to provide WASI imports to your plugin. +plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{}) + +if err != nil { + fmt.Printf("Failed to initialize plugin: %v\n", err) + os.Exit(1) +} +``` + +This plug-in was written in C and it does one thing, it counts vowels in a string. As such it exposes one "export" function: `count_vowels`. We can call exports using `Plugin.Call`: + +```go +exit, out, err := plugin.Call("count_vowels", data) +if err != nil { + fmt.Println(err) + os.Exit(int(exit)) +} + +response := string(out) + +// => {"count": 3, "total": 3, "vowels": "aeiouAEIOU"} +``` + +All exports have a simple interface of optional bytes in, and optional bytes out. This plug-in happens to take a string and return a JSON encoded string with a report of results. + +### Plug-in State + +Plug-ins may be stateful or stateless. Plug-ins can maintain state b/w calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export: + +```go +exit, out, err := plugin.Call("count_vowels", []byte("Hello, World!")) +if err != nil { + fmt.Println(err) + os.Exit(int(exit)) +} +// => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"} + +exit, out, err = plugin.Call("count_vowels", []byte("Hello, World!")) +if err != nil { + fmt.Println(err) + os.Exit(int(exit)) +} +// => {"count": 3, "total": 9, "vowels": "aeiouAEIOU"} +``` + +These variables will persist until this plug-in is freed or you initialize a new one. + +### Configuration + +Plug-ins may optionally take a configuration object. This is a static way to configure the plug-in. Our count-vowels plugin takes an optional configuration to change out which characters are considered vowels. Example: +```go +manifest := extism.Manifest{ + Wasm: []extism.Wasm{ + extism.WasmUrl{ + Url: "https://raw.githubusercontent.com/extism/extism/main/wasm/code.wasm", + }, + }, + Config: map[string]string{ + "vowels": "aeiouyAEIOUY", + }, +} + +ctx := context.Background() +config := extism.PluginConfig{ + EnableWasi: true, +} + +plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{}) + +if err != nil { + fmt.Printf("Failed to initialize plugin: %v\n", err) + os.Exit(1) +} + +exit, out, err := plugin.Call("count_vowels", []byte("Yellow, World!")) +if err != nil { + fmt.Println(err) + os.Exit(int(exit)) +} +// => {"count": 4, "total": 4, "vowels": "aeiouAEIOUY"} +``` + +### Host Functions + +Host functions can be a complicated concept. You can think of them like custom syscalls for your plug-in. You can use them to add capabilities to your plug-in through a simple interface. + +Another way to look at it is this: Up until now we've only invoked functions given to us by our plug-in, but what if our plug-in needs to invoke a function in our Go app? Host functions allow you to do this by passing a reference to a Go method to the plug-in. + +Let's load up a version of count vowels with a host function: + +```go +manifest := extism.Manifest{ + Wasm: []extism.Wasm{ + extism.WasmUrl{ + Url: "https://raw.githubusercontent.com/extism/extism/main/wasm/count-vowels-host.wasm", + }, + }, +} +``` + +Unlike our original plug-in, this plug-in expects you to provide your own implementation of "is_vowel" in Go. + +First let's write our host function: + +```go +hf := extism.NewHostFunctionWithStack( + "is_vowel", + "env", + func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) { + vowels := "AEIOUaeiou" + + result := 0 + + r := rune(api.DecodeI32(stack[0])) + if strings.ContainsRune(vowels, r) { + result = 1 + } + + stack[0] = api.EncodeI32(int32(result)) + }, + + // we need to give it the Wasm signature, it takes one i64 as input which acts as a pointer to a string + // and it returns an i64 which is the 0 or 1 result + []api.ValueType{api.ValueTypeI32}, + api.ValueTypeI32, +) +``` + +This method will be exposed to the plug-in in it's native language. We need to know the inputs and outputs and their types ahead of time. This function expects a string (single character) as the first input and expects a 0 (false) or 1 (true) in the output (returns). + +We need to pass these imports to the plug-in to create them. All imports of a plug-in must be satisfied for it to be initialized: + +```go +plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{hf}); + +exit, out, err := plugin.Call("count_vowels", []byte("Hello, World!")) + +// => {"count": 3, "total": 3} +``` + +Although this is a trivial example, you could imagine some more elaborate APIs for host functions. This is truly how you unleash the power of the plugin. You could, for example, imagine giving the plug-in access to APIs your app normally has like reading from a database, authenticating a user, sending messages, etc. + +## Build example plugins +Since our [example plugins](./plugins/) are also written in Go, for compiling them we use [TinyGo](https://tinygo.org/): ```sh cd plugins/config tinygo build -target wasi -o ../wasm/config.wasm main.go -``` +``` \ No newline at end of file From 1a00514967d7fab53f83f646cfc1b45baf52f086 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 19 Sep 2023 16:28:17 -0500 Subject: [PATCH 2/5] Add new count vowels plugin --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7886daa..7e66106 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For simplicity let's load one from the web: manifest := extism.Manifest{ Wasm: []extism.Wasm{ extism.WasmUrl{ - Url: "https://raw.githubusercontent.com/extism/extism/main/wasm/code.wasm", + Url: "https://github.com/extism/plugins/releases/latest/download/count_vowels.wasm", }, }, } @@ -94,7 +94,7 @@ Plug-ins may optionally take a configuration object. This is a static way to con manifest := extism.Manifest{ Wasm: []extism.Wasm{ extism.WasmUrl{ - Url: "https://raw.githubusercontent.com/extism/extism/main/wasm/code.wasm", + Url: "https://github.com/extism/plugins/releases/latest/download/count_vowels.wasm", }, }, Config: map[string]string{ @@ -187,4 +187,4 @@ Since our [example plugins](./plugins/) are also written in Go, for compiling th ```sh cd plugins/config tinygo build -target wasi -o ../wasm/config.wasm main.go -``` \ No newline at end of file +``` From 809eb0e5fafdb549526734479397bf3f4783368e Mon Sep 17 00:00:00 2001 From: Muhammad Azeez Date: Tue, 3 Oct 2023 18:10:50 +0300 Subject: [PATCH 3/5] fix: turn returnTypes into an array again to support void return types --- extism_test.go | 8 ++++---- host.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/extism_test.go b/extism_test.go index ffb543c..9cb01f7 100644 --- a/extism_test.go +++ b/extism_test.go @@ -234,7 +234,7 @@ func TestHost_simple(t *testing.T) { stack[0] = api.EncodeI32(a * b) }, []api.ValueType{api.ValueTypeI64, api.ValueTypeI64}, - api.ValueTypeI64, + []api.ValueType{api.ValueTypeI64}, ) if plugin, ok := plugin(t, manifest, mult); ok { @@ -277,7 +277,7 @@ func TestHost_memory(t *testing.T) { stack[0] = offset }, []api.ValueType{api.ValueTypeI64}, - api.ValueTypeI64, + []api.ValueType{api.ValueTypeI64}, ) if plugin, ok := plugin(t, manifest, mult); ok { @@ -325,7 +325,7 @@ func TestHost_multiple(t *testing.T) { stack[0] = offset }, []api.ValueType{api.ValueTypeI64}, - api.ValueTypeI64, + []api.ValueType{api.ValueTypeI64}, ) purple_message := NewHostFunctionWithStack( @@ -351,7 +351,7 @@ func TestHost_multiple(t *testing.T) { stack[0] = offset }, []api.ValueType{api.ValueTypeI64}, - api.ValueTypeI64, + []api.ValueType{api.ValueTypeI64}, ) hostFunctions := []HostFunction{ diff --git a/host.go b/host.go index dbd2485..3d3c592 100644 --- a/host.go +++ b/host.go @@ -77,14 +77,14 @@ func NewHostFunctionWithStack( namespace string, callback HostFunctionStackCallback, params []api.ValueType, - returnType api.ValueType) HostFunction { + returnTypes []api.ValueType) HostFunction { return HostFunction{ stackCallback: callback, Name: name, Namespace: namespace, Params: params, - Returns: []api.ValueType{returnType}, + Returns: returnTypes, } } From c9a81a8641127a49f799b55bc3852c3fd4c42e0c Mon Sep 17 00:00:00 2001 From: Muhammad Azeez Date: Tue, 3 Oct 2023 18:11:08 +0300 Subject: [PATCH 4/5] docs: update readme to be like the ruby-sdk --- README.md | 108 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 7e66106..d110418 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ # Extism Go SDK -> **Note**: This houses the 1.0 version of the Go SDK and is a work in progress. Please use the Go SDK in [extism/extism](https://github.com/extism/extism) until we hit 1.0. - This repo houses the Go SDK for integrating with the [Extism](https://extism.org/) runtime. Install this library into your host Go applications to run Extism plugins. Join the [Discord](https://discord.gg/EGTV8Pxs) and chat with us! +> **Note**: If you're unsure what Extism is or what an SDK is see our homepage: [https://extism.org](https://extism.org/). + +> **Note**: This houses the 1.0 version of the Go SDK and is a work in progress. Please use the Go SDK in [extism/extism](https://github.com/extism/extism) until we hit 1.0. + ## Installation Install via `go get`: @@ -16,14 +18,15 @@ go get github.com/extism/go-sdk ## Getting Started -The primary concept in Extism is the plug-in. You can think of a plug-in as a code module. It has imports and it has exports. These imports and exports define the interface, or your API. You decide what they are called and typed, and what they do. Then the plug-in developer implements them and you can call them. +This guide should walk you through some of the concepts in Extism and this Go library. + +### Creating A Plug-in -The code for a plug-in exist as a binary wasm module. We can load this with the raw bytes or we can use the manifest to tell Extism how to load it from disk or the web. +The primary concept in Extism is the [plug-in](https://extism.org/docs/concepts/plug-in). You can think of a plug-in as a code module stored in a `.wasm` file. -For simplicity let's load one from the web: +You'll normally load a plug-in from disk, but since you may not have one handy let's load a demo plug-in from the web: ```go -// NOTE: The schema for this manifest can be found here: https://extism.org/docs/concepts/manifest/ manifest := extism.Manifest{ Wasm: []extism.Wasm{ extism.WasmUrl{ @@ -37,9 +40,6 @@ config := extism.PluginConfig{ EnableWasi: true, } -// NOTE: if you encounter an error such as: -// "Unable to load plugin: unknown import: wasi_snapshot_preview1::fd_write has not been defined" -// make sure extism.PluginConfig.EnableWasi is set to `true` to provide WASI imports to your plugin. plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{}) if err != nil { @@ -47,8 +47,11 @@ if err != nil { os.Exit(1) } ``` +> **Note**: See [the Manifest docs](https://pkg.go.dev/github.com/extism/go-sdk#Manifest) as it has a rich schema and a lot of options. + +### Calling A Plug-in's Exports -This plug-in was written in C and it does one thing, it counts vowels in a string. As such it exposes one "export" function: `count_vowels`. We can call exports using `Plugin.Call`: +This plug-in was written in Rust and it does one thing, it counts vowels in a string. As such, it exposes one "export" function: `count_vowels`. We can call exports using [extism.Plugin.Call](https://pkg.go.dev/github.com/extism/go-sdk#Plugin.Call): ```go exit, out, err := plugin.Call("count_vowels", data) @@ -124,67 +127,102 @@ if err != nil { ### Host Functions -Host functions can be a complicated concept. You can think of them like custom syscalls for your plug-in. You can use them to add capabilities to your plug-in through a simple interface. +Let's extend our count-vowels example a little bit: Instead of storing the `total` in an ephemeral plug-in var, let's store it in a persistent key-value store! -Another way to look at it is this: Up until now we've only invoked functions given to us by our plug-in, but what if our plug-in needs to invoke a function in our Go app? Host functions allow you to do this by passing a reference to a Go method to the plug-in. +Wasm can't use our KV store on it's own. This is where [Host Functions](https://extism.org/docs/concepts/host-functions) come in. -Let's load up a version of count vowels with a host function: +[Host functions](https://extism.org/docs/concepts/host-functions) allow us to grant new capabilities to our plug-ins from our application. They are simply some Go functions you write which can be passed down and invoked from any language inside the plug-in. + +Let's load the manifest like usual but load up this `count_vowels_kvstore` plug-in: ```go manifest := extism.Manifest{ Wasm: []extism.Wasm{ extism.WasmUrl{ - Url: "https://raw.githubusercontent.com/extism/extism/main/wasm/count-vowels-host.wasm", + Url: "https://github.com/extism/plugins/releases/latest/download/count_vowels_kvstore.wasm", }, }, } ``` -Unlike our original plug-in, this plug-in expects you to provide your own implementation of "is_vowel" in Go. +> *Note*: The source code for this is [here](https://github.com/extism/plugins/blob/main/count_vowels_kvstore/src/lib.rs) and is written in rust, but it could be written in any of our PDK languages. -First let's write our host function: +Unlike our previous plug-in, this plug-in expects you to provide host functions that satisfy our its import interface for a KV store. +We want to expose two functions to our plugin, `kv_write(key string, value []bytes)` which writes a bytes value to a key and `kv_read(key string) []byte` which reads the bytes at the given `key`. ```go -hf := extism.NewHostFunctionWithStack( - "is_vowel", +// pretend this is Redis or something :) +kvStore := make(map[string][]byte) + +kvRead := extism.NewHostFunctionWithStack( + "kv_read", "env", func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) { - vowels := "AEIOUaeiou" - - result := 0 + key, err := p.ReadString(stack[0]) + if err != nil { + panic(err) + } - r := rune(api.DecodeI32(stack[0])) - if strings.ContainsRune(vowels, r) { - result = 1 + value, success := kvStore[key] + if !success { + value = []byte{0, 0, 0, 0} } - stack[0] = api.EncodeI32(int32(result)) + fmt.Printf("Read %v from key=%s\n", binary.LittleEndian.Uint32(value), key) + stack[0], err = p.WriteBytes(value) }, + []api.ValueType{api.ValueTypeI64}, + []api.ValueType{api.ValueTypeI64}, +) + +kvWrite := extism.NewHostFunctionWithStack( + "kv_write", + "env", + func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) { + key, err := p.ReadString(stack[0]) + if err != nil { + panic(err) + } + + value, err := p.ReadBytes(stack[1]) + if err != nil { + panic(err) + } - // we need to give it the Wasm signature, it takes one i64 as input which acts as a pointer to a string - // and it returns an i64 which is the 0 or 1 result - []api.ValueType{api.ValueTypeI32}, - api.ValueTypeI32, + fmt.Printf("Writing value=%v from key=%s\n", binary.LittleEndian.Uint32(value), key) + + kvStore[key] = value + }, + []api.ValueType{api.ValueTypeI64, api.ValueTypeI64}, + []api.ValueType{}, ) ``` -This method will be exposed to the plug-in in it's native language. We need to know the inputs and outputs and their types ahead of time. This function expects a string (single character) as the first input and expects a 0 (false) or 1 (true) in the output (returns). +> *Note*: In order to write host functions you should get familiar with the methods on the [extism.CurrentPlugin](https://pkg.go.dev/github.com/extism/go-sdk#CurrentPlugin) type. The `p` parameter is an instance of this type. We need to pass these imports to the plug-in to create them. All imports of a plug-in must be satisfied for it to be initialized: ```go -plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{hf}); +plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{kvRead, kvWrite}); +``` +Now we can invoke the event: + +```go exit, out, err := plugin.Call("count_vowels", []byte("Hello, World!")) +// => Read from key=count-vowels" +// => Writing value=3 from key=count-vowels" +// => {"count": 3, "total": 3, "vowels": "aeiouAEIOU"} -// => {"count": 3, "total": 3} +exit, out, err = plugin.Call("count_vowels", []byte("Hello, World!")) +// => Read from key=count-vowels" +// => Writing value=6 from key=count-vowels" +// => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"} ``` -Although this is a trivial example, you could imagine some more elaborate APIs for host functions. This is truly how you unleash the power of the plugin. You could, for example, imagine giving the plug-in access to APIs your app normally has like reading from a database, authenticating a user, sending messages, etc. - ## Build example plugins Since our [example plugins](./plugins/) are also written in Go, for compiling them we use [TinyGo](https://tinygo.org/): ```sh cd plugins/config tinygo build -target wasi -o ../wasm/config.wasm main.go -``` +``` \ No newline at end of file From 2e7b0fa02403e52a881443e2f265d5088b5ac8d8 Mon Sep 17 00:00:00 2001 From: Muhammad Azeez Date: Tue, 3 Oct 2023 18:12:47 +0300 Subject: [PATCH 5/5] add scheme url to manifest --- extism.go | 1 + 1 file changed, 1 insertion(+) diff --git a/extism.go b/extism.go index 5409c4f..a884451 100644 --- a/extism.go +++ b/extism.go @@ -204,6 +204,7 @@ func (u WasmUrl) ToWasmData(ctx context.Context) (WasmData, error) { } // Manifest represents the plugin's manifest, including Wasm modules and configuration. +// See https://extism.org/docs/concepts/manifest for schema. type Manifest struct { Wasm []Wasm `json:"wasm"` Memory struct {