From 4dd54a9186ad1bdb629d343175caa612315e89e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 27 Aug 2024 13:53:09 +0200 Subject: [PATCH] go-multi-rpc example --- examples/go/INSTRUCTIONS-mage | 14 + examples/go/go-multi-rpc/.gitignore | 2 + examples/go/go-multi-rpc/README.md | 188 ++++++++ .../go-multi-rpc/component-generator/main.go | 142 ++++++ .../component-template/component/main.go | 29 ++ .../component/wit/component.wit | 41 ++ .../components/component-one/main.go | 62 +++ .../component-one/wit/component-one.wit | 42 ++ .../pack-ns_component-three-stub/_stub.wit | 23 + .../component-three.wit | 41 ++ .../deps/pack-ns_component-two-stub/_stub.wit | 23 + .../pack-ns_component-two/component-two.wit | 41 ++ .../components/component-three/main.go | 29 ++ .../component-three/wit/component-three.wit | 41 ++ .../components/component-two/main.go | 49 ++ .../component-two/wit/component-two.wit | 41 ++ .../pack-ns_component-three-stub/_stub.wit | 23 + .../component-three.wit | 41 ++ examples/go/go-multi-rpc/go.mod | 15 + examples/go/go-multi-rpc/go.sum | 12 + .../integration/integration_test.go | 201 +++++++++ examples/go/go-multi-rpc/lib/cfg/cfg.go | 68 +++ examples/go/go-multi-rpc/mage.go | 14 + .../go/go-multi-rpc/magefiles/magefile.go | 423 ++++++++++++++++++ examples/go/go-multi-rpc/metadata.json | 17 + examples/go/go-multi-rpc/tools/tools.go | 6 + 26 files changed, 1628 insertions(+) create mode 100644 examples/go/INSTRUCTIONS-mage create mode 100644 examples/go/go-multi-rpc/.gitignore create mode 100644 examples/go/go-multi-rpc/README.md create mode 100644 examples/go/go-multi-rpc/component-generator/main.go create mode 100644 examples/go/go-multi-rpc/component-template/component/main.go create mode 100644 examples/go/go-multi-rpc/component-template/component/wit/component.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/main.go create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/component-one.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three-stub/_stub.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three/component-three.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two-stub/_stub.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two/component-two.wit create mode 100644 examples/go/go-multi-rpc/components/component-three/main.go create mode 100644 examples/go/go-multi-rpc/components/component-three/wit/component-three.wit create mode 100644 examples/go/go-multi-rpc/components/component-two/main.go create mode 100644 examples/go/go-multi-rpc/components/component-two/wit/component-two.wit create mode 100644 examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three-stub/_stub.wit create mode 100644 examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three/component-three.wit create mode 100644 examples/go/go-multi-rpc/go.mod create mode 100644 examples/go/go-multi-rpc/go.sum create mode 100644 examples/go/go-multi-rpc/integration/integration_test.go create mode 100644 examples/go/go-multi-rpc/lib/cfg/cfg.go create mode 100644 examples/go/go-multi-rpc/mage.go create mode 100644 examples/go/go-multi-rpc/magefiles/magefile.go create mode 100644 examples/go/go-multi-rpc/metadata.json create mode 100644 examples/go/go-multi-rpc/tools/tools.go diff --git a/examples/go/INSTRUCTIONS-mage b/examples/go/INSTRUCTIONS-mage new file mode 100644 index 0000000..22d4366 --- /dev/null +++ b/examples/go/INSTRUCTIONS-mage @@ -0,0 +1,14 @@ +See the documentation about installing tooling: https://learn.golem.cloud/docs/go-language-guide/setup +and see more details in the README.md file about other commands. + +Before first time build, generate RPC stubs: + go run mage.go updateRpcStubs + +Generate bindings, compile and compose components: + go run mage.go build + +Deploy components to default profile: + go run mage.go deploy + +Test the deployed components: + go run mage.go testIntegration diff --git a/examples/go/go-multi-rpc/.gitignore b/examples/go/go-multi-rpc/.gitignore new file mode 100644 index 0000000..c149d44 --- /dev/null +++ b/examples/go/go-multi-rpc/.gitignore @@ -0,0 +1,2 @@ +binding +/target/ diff --git a/examples/go/go-multi-rpc/README.md b/examples/go/go-multi-rpc/README.md new file mode 100644 index 0000000..2bbef76 --- /dev/null +++ b/examples/go/go-multi-rpc/README.md @@ -0,0 +1,188 @@ +# Golem Go Example with Multiple Components and Worker to Worker RPC Communication + +## Building +The project uses [magefile](https://magefile.org/) for building. Either install the tool binary `mage`, +or use the __zero install option__: `go run mage.go`. This readme will use the latter. + +To see the available commands use: + +```shell +go run mage.go +Targets: + addStubDependency adds generated and built stub dependency to componentGolemCliAddStubDependency + build alias for BuildAllComponents + buildAllComponents builds all components + buildComponent builds component by name + buildStubComponent builds RPC stub for component + clean cleans the projects + deploy adds or updates all the components with golem-cli\'s default profile + generateBinding generates go binding from WIT + generateNewComponent generates a new component based on the component-template + stubCompose composes dependencies + testIntegration tests the deployed components + tinyGoBuildComponentBinary build wasm component binary with tiny go + updateRpcStubs builds rpc stub components and adds them as dependency + wasmToolsComponentEmbed embeds type info into wasm component with wasm-tools + wasmToolsComponentNew create golem component with wasm-tools +``` + +For building the project for the first time (or after `clean`) use the following commands: + +```shell +go run mage.go updateRpcStubs +go run mage.go build +``` + +After this, using the `build` command is enough, unless there are changes in the RPC dependencies, +in that case `updateRpcStubs` is needed again. + +The final components that are usable by golem are placed in the `target/components` folder. + +## Deploying and testing the example + +In the example 3 simple counter components are defined, which can be familiar from the smaller examples. To showcase the remote calls, the counters `add` functions are connected, apart from increasing their own counter: + - **component one** delegates the add call to **component two** and **three** too, + - and **component two** delegates to **component three**. + +In both cases the _current worker name_ will be used as _target worker name_ too. + +Apart from _worker name_, remote calls also require the **target components' deployed ID**. For this the example uses environment variables, and uses the `lib/cfg` subpackage (which is shared between the components) to extract it. + +The examples assume a configured default `golem-cli` profile, and will use that. + +To test, first we have to build the project as seen in the above: + +```shell +go run mage.go updateRpcStubs +go run mage.go build +``` + +Then we can deploy our components with `golem-cli`, for which a wrapper magefile command is provided: + +```shell +go run mage.go deploy +``` + +Once the components are deployed, a simple example integration test suite can be used to test the components. +The tests are in the [/integration/integration_test.go](/integration/integration_test.go) test file, and can be run with: + +```shell +go run mage.go testIntegration +``` + +The `TestDeployed` simply tests if our components metadata is available through `golem-cli component get`. + +The `TestCallingAddOnComponentOneCallsToOtherComponents` will: + - get the _component URNs_ with `golem-cli component get` + - generates a _random worker name_, so our tests are starting from a clean state + - adds 1 - 1 worker for component one and component two with the required _environment variables_ containing the other workers' _component ids_ + - then makes various component invocations with `golem-cli worker invoke-and-await` and tests if the counters - after increments - are holding the right value according to the delegated `add` function calls. + +## Adding Components + +Use the `generateNewComponent` command to add new components to the project: + +```shell +go run mage.go generateNewComponent component-four +``` + +The above will create a new component in the `components/component-four` directory based on the template at [/component-template/component](/component-template/component). + +After adding a new component the `build` command will also include it. + +## Using Worker to Worker RPC calls + +### Under the hood + +Under the hood the _magefile_ commands below (and for build) use generic `golem-cli stubgen` subcommands: + - `golem-cli stubgen build` for creating remote call _stub WIT_ definitions and _WASM components_ for the stubs + - `golem-cli stubgen add-stub-dependency` for adding the _stub WIT_ definitions to a _component's WIT_ dependencies + - `golem-cli stubgen compose` for _composing_ components with the stub components + +### Magefile commands and required manual steps + +The dependencies between components are defined in the [/magefiles/magefile.go](/magefiles/magefile.go) build script: + +```go +// componentDeps defines the Worker to Worker RPC dependencies +var componentDeps = map[string][]string{ + "component-one": {"component-two", "component-three"}, + "component-two": {"component-three"}, +} +``` + +After changing dependencies the `updateRpcStubs` command can be used to create the necessary stubs: + +```shell +go run mage.go updateRpcStubs +``` + +The command will create stubs for the dependency projects in the ``/target/stub`` directory and will also place the required stub _WIT_ interfaces on the dependant component's `wit/deps` directory. + +To actually use the dependencies in a project it also has to be manually imported in the component's world. + +E.g. with the above definitions the following import has to be __manually__ added to `/components/component-one/wit/component-one.wit`: + +```wit +import pack-ns:component-two-stub; +import pack-ns:component-three-stub; +``` + +So the component definition should like similar to this: + +```wit +package pack-ns:component-one; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-one-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-one { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + // . + // . + // . + // other dependencies + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + import pack-ns:component-two-stub; + import pack-ns:component-three-stub; + + export component-one-api; +} +``` + +After this `build` (or the `generateBinding`) command can be used to update bindings, which now should include the +required functions for calling other components. + +Here's an example that delegates the `Add` call to another component and waits for the result: + +```go +import ( + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/component-one/binding" +) + + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + componentTwo := binding.NewComponentTwoApi(binding.GolemRpc0_1_0_TypesUri{Value: "uri"}) + defer componentTwo.Drop() + componentTwo.BlockingAdd(value) + + i.counter += value +} +``` + +Once a remote call is in place, the `build` command will also compose the stub components into the caller component. diff --git a/examples/go/go-multi-rpc/component-generator/main.go b/examples/go/go-multi-rpc/component-generator/main.go new file mode 100644 index 0000000..c5e7f4b --- /dev/null +++ b/examples/go/go-multi-rpc/component-generator/main.go @@ -0,0 +1,142 @@ +package main + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +func main() { + if len(os.Args) != 3 { + exit(0, fmt.Sprintf("Usage: %s ", os.Args[0])) + } + + componentTemplateRoot := "component-template/component" + pkgOrg := os.Args[1] + compName := os.Args[2] + componentDir := filepath.Join("components", compName) + + _, err := os.Stat(componentDir) + if err == nil { + exit(1, fmt.Sprintf("Target component directory already exists: %s", componentDir)) + } + if err != nil && !os.IsNotExist(err) { + exit(1, err.Error()) + } + + err = os.MkdirAll(componentDir, 0755) + if err != nil { + exit(1, err.Error()) + } + + err = fs.WalkDir( + os.DirFS(componentTemplateRoot), + ".", + func(path string, d fs.DirEntry, err error) error { + srcFilePath := filepath.Join(componentTemplateRoot, path) + fileInfo, err := os.Stat(srcFilePath) + if err != nil { + return fmt.Errorf("stat failed for template %s, %w", srcFilePath, err) + } + + if fileInfo.IsDir() { + return nil + } + + switch path { + case "main.go": + err = generateFile(pkgOrg, compName, srcFilePath, filepath.Join(componentDir, path)) + case "wit/component.wit": + err = generateFile(pkgOrg, compName, srcFilePath, filepath.Join(componentDir, "wit", compName+".wit")) + default: + err = copyFile(srcFilePath, filepath.Join(componentDir, path)) + } + if err != nil { + return fmt.Errorf("template generation failed for %s, %w", srcFilePath, err) + } + + return nil + }) + if err != nil { + exit(1, err.Error()) + } +} + +func generateFile(pkgOrg, compName, srcFileName, dstFileName string) error { + pascalPkgOrg := dashToPascal(pkgOrg) + pascalCompName := dashToPascal(compName) + + fmt.Printf("Generating from %s to %s\n", srcFileName, dstFileName) + + contentsBs, err := os.ReadFile(srcFileName) + if err != nil { + return fmt.Errorf("generateFile: read file failed for %s, %w", srcFileName, err) + } + + contents := string(contentsBs) + + contents = strings.ReplaceAll(contents, "comp-name", compName) + contents = strings.ReplaceAll(contents, "pck-ns", pkgOrg) + contents = strings.ReplaceAll(contents, "CompName", pascalCompName) + contents = strings.ReplaceAll(contents, "PckNs", pascalPkgOrg) + + dstDir := filepath.Dir(dstFileName) + err = os.MkdirAll(dstDir, 0755) + if err != nil { + return fmt.Errorf("generateFile: mkdir failed for %s, %w", dstDir, err) + } + + err = os.WriteFile(dstFileName, []byte(contents), 0644) + if err != nil { + return fmt.Errorf("generateFile: write file failed for %s, %w", dstFileName, err) + } + + return nil +} + +func dashToPascal(s string) string { + parts := strings.Split(s, "-") + for i, part := range parts { + if len(part) > 0 { + parts[i] = strings.ToUpper(string(part[0])) + part[1:] + } + } + return strings.Join(parts, "") +} + +func copyFile(srcFileName, dstFileName string) error { + fmt.Printf("Copy %s to %s\n", srcFileName, dstFileName) + + src, err := os.Open(srcFileName) + if err != nil { + return fmt.Errorf("copyFile: open failed for %s, %w", srcFileName, err) + } + defer func() { _ = src.Close() }() + + dstDir := filepath.Dir(dstFileName) + err = os.MkdirAll(dstDir, 0755) + if err != nil { + return fmt.Errorf("copyFile: mkdir failed for %s, %w", dstDir, err) + } + + dst, err := os.Create(dstFileName) + if err != nil { + return fmt.Errorf("copyFile: create file failed for %s, %w", dstFileName, err) + } + defer func() { _ = dst.Close() }() + + _, err = io.Copy(dst, src) + if err != nil { + return fmt.Errorf("copyFile: copy failed from %s to %s, %w", srcFileName, dstFileName, err) + } + + return nil +} + +func exit(code int, message string) { + fmt.Println(message) + os.Exit(code) +} diff --git a/examples/go/go-multi-rpc/component-template/component/main.go b/examples/go/go-multi-rpc/component-template/component/main.go new file mode 100644 index 0000000..99b55da --- /dev/null +++ b/examples/go/go-multi-rpc/component-template/component/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/comp-name/binding" +) + +func init() { + binding.SetExportsPckNsCompNameCompNameApi(&Impl{}) +} + +type Impl struct { + counter uint64 +} + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + i.counter += value +} + +func (i *Impl) Get() uint64 { + std.Init(std.Packages{Os: true, NetHttp: true}) + + return i.counter +} + +func main() {} diff --git a/examples/go/go-multi-rpc/component-template/component/wit/component.wit b/examples/go/go-multi-rpc/component-template/component/wit/component.wit new file mode 100644 index 0000000..1e7b6cf --- /dev/null +++ b/examples/go/go-multi-rpc/component-template/component/wit/component.wit @@ -0,0 +1,41 @@ +package pck-ns:comp-name; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface comp-name-api { + add: func(value: u64); + get: func() -> u64; +} + +world comp-name { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + // import pck-ns:name-stub/stub-name; + + export comp-name-api; +} diff --git a/examples/go/go-multi-rpc/components/component-one/main.go b/examples/go/go-multi-rpc/components/component-one/main.go new file mode 100644 index 0000000..02658fe --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/main.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + + "github.com/golemcloud/golem-go/golemhost" + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/component-one/binding" + // NOTE: use the lib folder to create common packages used by multiple components + "golem-go-project/lib/cfg" +) + +func init() { + binding.SetExportsPackNsComponentOneComponentOneApi(&Impl{}) +} + +type Impl struct { + counter uint64 +} + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + selfWorkerName := golemhost.GetSelfMetadata().WorkerId.WorkerName + + { + componentTwoWorkerURI, err := cfg.ComponentTwoWorkerURI(selfWorkerName) + if err != nil { + fmt.Printf("%+v\n", err) + return + } + + fmt.Printf("Calling %s...\n", componentTwoWorkerURI.Value) + componentTwo := binding.NewComponentTwoApi(binding.GolemRpc0_1_0_TypesUri(componentTwoWorkerURI)) + defer componentTwo.Drop() + componentTwo.BlockingAdd(value) + } + + { + componentThreeWorkerURI, err := cfg.ComponentThreeWorkerURI(selfWorkerName) + if err != nil { + fmt.Printf("%+v\n", err) + return + } + + fmt.Printf("Calling %s...\n", componentThreeWorkerURI.Value) + componentThree := binding.NewComponentThreeApi(binding.GolemRpc0_1_0_TypesUri(componentThreeWorkerURI)) + defer componentThree.Drop() + componentThree.BlockingAdd(value) + } + + i.counter += value +} + +func (i *Impl) Get() uint64 { + std.Init(std.Packages{Os: true, NetHttp: true}) + + return i.counter +} + +func main() {} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/component-one.wit b/examples/go/go-multi-rpc/components/component-one/wit/component-one.wit new file mode 100644 index 0000000..d9befa1 --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/component-one.wit @@ -0,0 +1,42 @@ +package pack-ns:component-one; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-one-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-one { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + import pack-ns:component-two-stub/stub-component-two; + import pack-ns:component-three-stub/stub-component-three; + + export component-one-api; +} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three-stub/_stub.wit b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three-stub/_stub.wit new file mode 100644 index 0000000..ab3de6b --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three-stub/_stub.wit @@ -0,0 +1,23 @@ +package pack-ns:component-three-stub; + +interface stub-component-three { + use golem:rpc/types@0.1.0.{uri as golem-rpc-uri}; + use wasi:io/poll@0.2.0.{pollable as wasi-io-pollable}; + + resource future-get-result { + subscribe: func() -> wasi-io-pollable; + get: func() -> option; + } + resource component-three-api { + constructor(location: golem-rpc-uri); + blocking-add: func(value: u64); + add: func(value: u64); + blocking-get: func() -> u64; + get: func() -> future-get-result; + } + +} + +world wasm-rpc-stub-component-three { + export stub-component-three; +} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three/component-three.wit b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three/component-three.wit new file mode 100644 index 0000000..7bd3d9c --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three/component-three.wit @@ -0,0 +1,41 @@ +package pack-ns:component-three; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-three-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-three { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + // e.g: import pack-ns:name-stub/stub-name; + + export component-three-api; +} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two-stub/_stub.wit b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two-stub/_stub.wit new file mode 100644 index 0000000..f83aa85 --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two-stub/_stub.wit @@ -0,0 +1,23 @@ +package pack-ns:component-two-stub; + +interface stub-component-two { + use golem:rpc/types@0.1.0.{uri as golem-rpc-uri}; + use wasi:io/poll@0.2.0.{pollable as wasi-io-pollable}; + + resource future-get-result { + subscribe: func() -> wasi-io-pollable; + get: func() -> option; + } + resource component-two-api { + constructor(location: golem-rpc-uri); + blocking-add: func(value: u64); + add: func(value: u64); + blocking-get: func() -> u64; + get: func() -> future-get-result; + } + +} + +world wasm-rpc-stub-component-two { + export stub-component-two; +} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two/component-two.wit b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two/component-two.wit new file mode 100644 index 0000000..bb81e6d --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two/component-two.wit @@ -0,0 +1,41 @@ +package pack-ns:component-two; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-two-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-two { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + //import pack-ns:component-three-stub/stub-component-three; + + export component-two-api; +} diff --git a/examples/go/go-multi-rpc/components/component-three/main.go b/examples/go/go-multi-rpc/components/component-three/main.go new file mode 100644 index 0000000..646e5f1 --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-three/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/component-three/binding" +) + +func init() { + binding.SetExportsPackNsComponentThreeComponentThreeApi(&Impl{}) +} + +type Impl struct { + counter uint64 +} + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + i.counter += value +} + +func (i *Impl) Get() uint64 { + std.Init(std.Packages{Os: true, NetHttp: true}) + + return i.counter +} + +func main() {} diff --git a/examples/go/go-multi-rpc/components/component-three/wit/component-three.wit b/examples/go/go-multi-rpc/components/component-three/wit/component-three.wit new file mode 100644 index 0000000..7bd3d9c --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-three/wit/component-three.wit @@ -0,0 +1,41 @@ +package pack-ns:component-three; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-three-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-three { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + // e.g: import pack-ns:name-stub/stub-name; + + export component-three-api; +} diff --git a/examples/go/go-multi-rpc/components/component-two/main.go b/examples/go/go-multi-rpc/components/component-two/main.go new file mode 100644 index 0000000..8614a27 --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-two/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + + "github.com/golemcloud/golem-go/golemhost" + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/component-two/binding" + // NOTE: use the lib folder to create common packages used by multiple components + "golem-go-project/lib/cfg" +) + +func init() { + binding.SetExportsPackNsComponentTwoComponentTwoApi(&Impl{}) +} + +type Impl struct { + counter uint64 +} + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + selfWorkerName := golemhost.GetSelfMetadata().WorkerId.WorkerName + + { + componentThreeWorkerURI, err := cfg.ComponentThreeWorkerURI(selfWorkerName) + if err != nil { + fmt.Printf("%+v\n", err) + return + } + + fmt.Printf("Calling %s...\n", componentThreeWorkerURI.Value) + componentThree := binding.NewComponentThreeApi(binding.GolemRpc0_1_0_TypesUri(componentThreeWorkerURI)) + defer componentThree.Drop() + componentThree.BlockingAdd(value) + } + + i.counter += value +} + +func (i *Impl) Get() uint64 { + std.Init(std.Packages{Os: true, NetHttp: true}) + + return i.counter +} + +func main() {} diff --git a/examples/go/go-multi-rpc/components/component-two/wit/component-two.wit b/examples/go/go-multi-rpc/components/component-two/wit/component-two.wit new file mode 100644 index 0000000..6ed354b --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-two/wit/component-two.wit @@ -0,0 +1,41 @@ +package pack-ns:component-two; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-two-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-two { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + import pack-ns:component-three-stub/stub-component-three; + + export component-two-api; +} diff --git a/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three-stub/_stub.wit b/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three-stub/_stub.wit new file mode 100644 index 0000000..ab3de6b --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three-stub/_stub.wit @@ -0,0 +1,23 @@ +package pack-ns:component-three-stub; + +interface stub-component-three { + use golem:rpc/types@0.1.0.{uri as golem-rpc-uri}; + use wasi:io/poll@0.2.0.{pollable as wasi-io-pollable}; + + resource future-get-result { + subscribe: func() -> wasi-io-pollable; + get: func() -> option; + } + resource component-three-api { + constructor(location: golem-rpc-uri); + blocking-add: func(value: u64); + add: func(value: u64); + blocking-get: func() -> u64; + get: func() -> future-get-result; + } + +} + +world wasm-rpc-stub-component-three { + export stub-component-three; +} diff --git a/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three/component-three.wit b/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three/component-three.wit new file mode 100644 index 0000000..7bd3d9c --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three/component-three.wit @@ -0,0 +1,41 @@ +package pack-ns:component-three; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-three-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-three { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + // e.g: import pack-ns:name-stub/stub-name; + + export component-three-api; +} diff --git a/examples/go/go-multi-rpc/go.mod b/examples/go/go-multi-rpc/go.mod new file mode 100644 index 0000000..5040c43 --- /dev/null +++ b/examples/go/go-multi-rpc/go.mod @@ -0,0 +1,15 @@ +module golem-go-project + +go 1.20.0 + +require ( + github.com/golemcloud/golem-go v0.7.0 + github.com/google/uuid v1.6.0 + github.com/magefile/mage v1.15.0 + github.com/tidwall/gjson v1.17.3 +) + +require ( + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect +) diff --git a/examples/go/go-multi-rpc/go.sum b/examples/go/go-multi-rpc/go.sum new file mode 100644 index 0000000..95966ad --- /dev/null +++ b/examples/go/go-multi-rpc/go.sum @@ -0,0 +1,12 @@ +github.com/golemcloud/golem-go v0.7.0 h1:8zpNeAtEDJsJllqHUmjji/lyplSJ0kI3v7vN97/N8zs= +github.com/golemcloud/golem-go v0.7.0/go.mod h1:VLL22qVo5R2+jGLO43tLPpPjf2WrA6B7GQoqXKnSODo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/examples/go/go-multi-rpc/integration/integration_test.go b/examples/go/go-multi-rpc/integration/integration_test.go new file mode 100644 index 0000000..1be0d0a --- /dev/null +++ b/examples/go/go-multi-rpc/integration/integration_test.go @@ -0,0 +1,201 @@ +package integration + +import ( + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/magefile/mage/sh" + "github.com/tidwall/gjson" +) + +func TestDeployed(t *testing.T) { + componentOneURN := mustGetCompURNByCompName(t, "component-one") + fmt.Printf("component-one: %s\n", componentOneURN) + + componentTwoURN := mustGetCompURNByCompName(t, "component-two") + fmt.Printf("component-two: %s\n", componentTwoURN) + + componentThreeURN := mustGetCompURNByCompName(t, "component-three") + fmt.Printf("component-three: %s\n", componentThreeURN) +} + +func TestCallingAddOnComponentOneCallsToOtherComponents(t *testing.T) { + workerName := uuid.New().String() + fmt.Printf("random worker name for test: %s\n", workerName) + + // Setup + { + // Get all component URNs + componentURNs := mustGetComponentURNs(t) + fmt.Printf("component urns: %#v\n", componentURNs) + + // Preparing workers with env vars for RPC, so they know the other component IDs + mustAddComponent(t, "component-one", workerName, componentURNs) + mustAddComponent(t, "component-two", workerName, componentURNs) + } + + // Call get on all component and check the counters are 0 + { + expectCounter(t, "component-one", workerName, 0) + expectCounter(t, "component-two", workerName, 0) + expectCounter(t, "component-three", workerName, 0) + } + + // Invoke add on component-one + { + mustInvokeAndAwaitComponent(t, "component-one", workerName, "pack-ns:component-one/component-one-api.{add}", []string{"3"}) + } + + // Call get on all component and check the counters are accumulated on component two and three + { + expectCounter(t, "component-one", workerName, 3) + expectCounter(t, "component-two", workerName, 3) + expectCounter(t, "component-three", workerName, 6) + } + + // Invoke add on component-two + { + mustInvokeAndAwaitComponent(t, "component-two", workerName, "pack-ns:component-two/component-two-api.{add}", []string{"2"}) + } + + // Call get on all component and check the counters are accumulated on component three + { + expectCounter(t, "component-one", workerName, 3) + expectCounter(t, "component-two", workerName, 5) + expectCounter(t, "component-three", workerName, 8) + } + + // Invoke add on component-one again + { + mustInvokeAndAwaitComponent(t, "component-one", workerName, "pack-ns:component-one/component-one-api.{add}", []string{"1"}) + } + + // Call get on all component and check the counters are accumulated on component two and three + { + expectCounter(t, "component-one", workerName, 4) + expectCounter(t, "component-two", workerName, 6) + expectCounter(t, "component-three", workerName, 10) + } +} + +func getCompURNByCompName(compName string) (string, error) { + output, err := sh.Output( + "golem-cli", "--format", "json", "component", "get", "--component"+"-name", compName, + ) + if err != nil { + return "", fmt.Errorf("getCompURNByCompName for %s: golem-cli failed: %w\n", compName, err) + } + + componentURN := gjson.Get(output, "componentUrn").String() + if componentURN == "" { + return "", fmt.Errorf("missing componentURN in response:\n%s\n", output) + } + + return componentURN, nil +} + +func mustGetCompURNByCompName(t *testing.T, compName string) string { + name, err := getCompURNByCompName(compName) + if err != nil { + t.Fatalf("%+v", err) + } + return name +} + +type ComponentURNs struct { + ComponentOne string + ComponentTwo string + ComponentThree string +} + +func mustGetComponentURNs(t *testing.T) ComponentURNs { + return ComponentURNs{ + ComponentOne: mustGetCompURNByCompName(t, "component-one"), + ComponentTwo: mustGetCompURNByCompName(t, "component-two"), + ComponentThree: mustGetCompURNByCompName(t, "component-three"), + } +} + +func addComponent(compName, workerName string, componentURNs ComponentURNs) error { + fmt.Printf("adding component: %s, %s\n", compName, workerName) + output, err := sh.Output( + "golem-cli", "worker", + "--format", "json", + "add", + "--component"+"-name", compName, + "--worker-name", workerName, + "--env", fmt.Sprintf("COMPONENT_ONE_ID=%s", componentIDFromURN(componentURNs.ComponentOne)), + "--env", fmt.Sprintf("COMPONENT_TWO_ID=%s", componentIDFromURN(componentURNs.ComponentTwo)), + "--env", fmt.Sprintf("COMPONENT_THREE_ID=%s", componentIDFromURN(componentURNs.ComponentThree)), + ) + if err != nil { + return fmt.Errorf("addComponent for %s, %s: golem-cli failed: %w\n%s", compName, workerName, err, output) + } + return nil +} + +func mustAddComponent(t *testing.T, compName, workerName string, componentURNs ComponentURNs) { + err := addComponent(compName, workerName, componentURNs) + if err != nil { + t.Fatalf("%+v", err) + } +} + +func invokeAndAwaitComponent(compName, workerName, function string, functionArgs []string) (string, error) { + fmt.Printf("invoking component: %s, %s, %s, %+v\n", compName, workerName, function, functionArgs) + + cliArgs := []string{ + "--format", "json", + "worker", + "invoke-and-await", + "--component" + "-name", compName, + "--worker-name", workerName, + "--function", function, + } + + for _, arg := range functionArgs { + cliArgs = append(cliArgs, []string{"--arg", arg}...) + } + + output, err := sh.Output("golem-cli", cliArgs...) + if err != nil { + return "", fmt.Errorf("invokeAndAwaitComponent failed: %w", err) + } + + fmt.Println(output) + + return output, nil +} + +func mustInvokeAndAwaitComponent(t *testing.T, componentURN, workerName, function string, functionArgs []string) string { + output, err := invokeAndAwaitComponent(componentURN, workerName, function, functionArgs) + if err != nil { + t.Fatalf("%+v", err) + } + return output +} + +func componentIDFromURN(urn string) string { + return strings.Split(urn, ":")[2] +} + +func expectCounter(t *testing.T, compName, workerName string, expected int64) { + output := mustInvokeAndAwaitComponent(t, compName, workerName, fmt.Sprintf("pack-ns:%s/%s-api.{get}", compName, compName), nil) + + actualValue := gjson.Get(output, "value") + if !actualValue.Exists() { + t.Fatalf("Expected counter for %s, %s: %d, actual value is missing", compName, workerName, expected) + } + + actualArray := actualValue.Array() + if len(actualArray) != 1 { + t.Fatalf("Expected counter for %s, %s: %d, actual value tuple has bad number of elements: %s", compName, workerName, expected, actualValue) + } + + actual := actualArray[0].Int() + if expected != actual { + t.Fatalf("Expected counter for %s, %s: %d, actual: %d", compName, workerName, expected, actual) + } +} diff --git a/examples/go/go-multi-rpc/lib/cfg/cfg.go b/examples/go/go-multi-rpc/lib/cfg/cfg.go new file mode 100644 index 0000000..f6bc382 --- /dev/null +++ b/examples/go/go-multi-rpc/lib/cfg/cfg.go @@ -0,0 +1,68 @@ +package cfg + +import ( + "fmt" + "os" + "strings" + + "github.com/google/uuid" + + "github.com/golemcloud/golem-go/binding" + "github.com/golemcloud/golem-go/golemhost" +) + +func ComponentIDFromEnv(key string) (golemhost.ComponentID, error) { + value := os.Getenv(key) + if value == "" { + return [16]byte{}, fmt.Errorf("missing environment variable for component id: %s", key) + } + componentID, err := uuid.Parse(strings.ToLower(value)) + if err != nil { + return [16]byte{}, fmt.Errorf("component id parse failed for %s=%s, %w", key, value, err) + } + return golemhost.ComponentID(componentID), nil +} + +func ComponentOneID() (golemhost.ComponentID, error) { + return ComponentIDFromEnv("COMPONENT_ONE_ID") +} + +func ComponentTwoID() (golemhost.ComponentID, error) { + return ComponentIDFromEnv("COMPONENT_TWO_ID") +} + +func ComponentThreeID() (golemhost.ComponentID, error) { + return ComponentIDFromEnv("COMPONENT_THREE_ID") +} + +func ComponentOneWorkerURI(workerName string) (binding.GolemRpc0_1_0_TypesUri, error) { + uri, err := workerURIF(ComponentOneID, workerName) + return uri, err +} + +func ComponentTwoWorkerURI(workerName string) (binding.GolemRpc0_1_0_TypesUri, error) { + uri, err := workerURIF(ComponentTwoID, workerName) + return uri, err +} + +func ComponentThreeWorkerURI(workerName string) (binding.GolemRpc0_1_0_TypesUri, error) { + uri, err := workerURIF(ComponentThreeID, workerName) + return uri, err +} + +func WorkerURI[T binding.GolemRpc0_1_0_TypesUri](workerID golemhost.WorkerID) T { + return T(binding.GolemRpc0_1_0_TypesUri{ + Value: fmt.Sprintf("urn:worker:%s/%s", (uuid.UUID(workerID.ComponentID)).URN(), workerID.WorkerName), + }) +} + +func workerURIF(getComponentID func() (golemhost.ComponentID, error), workerName string) (binding.GolemRpc0_1_0_TypesUri, error) { + componentID, err := getComponentID() + if err != nil { + return binding.GolemRpc0_1_0_TypesUri{}, err + } + return WorkerURI(golemhost.WorkerID{ + ComponentID: componentID, + WorkerName: workerName, + }), nil +} diff --git a/examples/go/go-multi-rpc/mage.go b/examples/go/go-multi-rpc/mage.go new file mode 100644 index 0000000..53c564a --- /dev/null +++ b/examples/go/go-multi-rpc/mage.go @@ -0,0 +1,14 @@ +//go:build ignore +// +build ignore + +package main + +import ( + "os" + + "github.com/magefile/mage/mage" +) + +func main() { + os.Exit(mage.Main()) +} diff --git a/examples/go/go-multi-rpc/magefiles/magefile.go b/examples/go/go-multi-rpc/magefiles/magefile.go new file mode 100644 index 0000000..61ac112 --- /dev/null +++ b/examples/go/go-multi-rpc/magefiles/magefile.go @@ -0,0 +1,423 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/magefile/mage/sh" + "github.com/magefile/mage/target" +) + +// componentDeps defines the Worker to Worker RPC dependencies +var componentDeps = map[string][]string{ + "component-one": {"component-two", "component-three"}, + "component-two": {"component-three"}, +} + +var pkgNs = "pack-ns" +var targetDir = "target" +var componentsDir = "components" +var libDir = "lib" +var wasiSnapshotPreview1Adapter = "adapters/tier1/wasi_snapshot_preview1.wasm" + +// Build alias for BuildAllComponents +func Build() error { + return BuildAllComponents() +} + +// BuildAllComponents builds all components +func BuildAllComponents() error { + for _, compName := range compNames() { + err := BuildComponent(compName) + if err != nil { + return fmt.Errorf("build all components: build component failed for %s, %w", compName, err) + } + } + + return nil +} + +// UpdateRpcStubs builds rpc stub components and adds them as dependency +func UpdateRpcStubs() error { + for _, compName := range stubCompNames() { + err := BuildStubComponent(compName) + if err != nil { + return fmt.Errorf("update RPC stubs: build stub component failed for %s, %w", compName, err) + } + } + + for _, compName := range compNames() { + for _, dependency := range componentDeps[compName] { + err := AddStubDependency(compName, dependency) + if err != nil { + return fmt.Errorf("update RPC stubs: add stub dependecy failed for %s to %s, %w", dependency, compName, err) + } + } + } + + return nil +} + +// BuildStubComponent builds RPC stub for component +func BuildStubComponent(compName string) error { + componentDir := filepath.Join(componentsDir, compName) + srcWitDir := filepath.Join(componentDir, "wit") + stubTargetDir := filepath.Join(targetDir, "stub", compName) + destWasm := filepath.Join(stubTargetDir, "stub.wasm") + destWitDir := filepath.Join(stubTargetDir, "wit") + + return opRun(op{ + RunMessage: fmt.Sprintf("Building stub component for %s", compName), + SkipMessage: "stub component build", + Targets: []string{destWasm, destWitDir}, + SourcePaths: []string{srcWitDir}, + Run: func() error { + return sh.RunV( + "golem-cli", "stubgen", "build", + "--source-wit-root", srcWitDir, + "--dest-wasm", destWasm, + "--dest-wit-root", destWitDir, + ) + }, + }) +} + +// AddStubDependency adds generated and built stub dependency to componentGolemCliAddStubDependency +func AddStubDependency(compName, depCompName string) error { + stubTargetDir := filepath.Join(targetDir, "stub", depCompName) + srcWitDir := filepath.Join(stubTargetDir, "wit") + dstComponentDir := filepath.Join(componentsDir, compName) + dstWitDir := filepath.Join(dstComponentDir, "wit") + dstWitDepDir := filepath.Join(dstComponentDir, dstWitDir, "deps", fmt.Sprintf("%s_%s", pkgNs, compName)) + dstWitDepStubDir := filepath.Join(dstComponentDir, dstWitDir, "deps", fmt.Sprintf("%s_%s-stub", pkgNs, compName)) + + return opRun(op{ + RunMessage: fmt.Sprintf("Adding stub dependecy for %s to %s", depCompName, compName), + SkipMessage: "add stub dependency", + Targets: []string{dstWitDepDir, dstWitDepStubDir}, + SourcePaths: []string{srcWitDir}, + Run: func() error { + return sh.RunV( + "golem-cli", "stubgen", "add-stub-dependency", + "--overwrite", + "--stub-wit-root", srcWitDir, + "--dest-wit-root", dstWitDir, + ) + }, + }) +} + +// StubCompose composes dependencies +func StubCompose(compName, componentWasm, targetWasm string) error { + buildTargetDir := filepath.Dir(componentWasm) + dependencies := componentDeps[compName] + + stubWasms := make([]string, len(dependencies)) + for i, compName := range dependencies { + stubTargetDir := filepath.Join(targetDir, "stub", compName) + stubWasms[i] = filepath.Join(stubTargetDir, "stub.wasm") + } + + return opRun(op{ + RunMessage: fmt.Sprintf("Composing %s into %s", fmt.Sprintf("[%s]", strings.Join(stubWasms, ", ")), compName), + SkipMessage: "composing", + Targets: []string{targetWasm}, + SourcePaths: append(stubWasms, componentWasm), + Run: func() error { + composeWasm := componentWasm + if len(stubWasms) > 0 { + srcWasm := componentWasm + for i, stubWasm := range stubWasms { + prevComposeWasm := composeWasm + composeWasm = filepath.Join( + buildTargetDir, + fmt.Sprintf("compose-%d-%s.wasm", i+1, filepath.Base(dependencies[i])), + ) + + outBuf := &bytes.Buffer{} + errBuff := &bytes.Buffer{} + + _, err := sh.Exec( + nil, outBuf, errBuff, + "golem-cli", "stubgen", "compose", + "--source-wasm", srcWasm, + "--stub-wasm", stubWasm, + "--dest-wasm", composeWasm, + ) + if err != nil { + errString := errBuff.String() + if strings.Contains(errString, "Error: no dependencies of component") && + strings.Contains(errString, "were found") { + fmt.Printf("Skipping composing %s, not used\n", stubWasm) + composeWasm = prevComposeWasm + continue + } + + fmt.Print(outBuf) + fmt.Print(errBuff) + + return fmt.Errorf("StubCompose failed: %w", err) + } + srcWasm = composeWasm + } + } + + return copyFile(composeWasm, targetWasm) + }, + }) +} + +// BuildComponent builds component by name +func BuildComponent(compName string) error { + componentDir := filepath.Join(componentsDir, compName) + witDir := filepath.Join(componentDir, "wit") + bindingDir := filepath.Join(componentDir, "binding") + buildTargetDir := filepath.Join(targetDir, "build", compName) + componentsTargetDir := filepath.Join(targetDir, "components") + moduleWasm := filepath.Join(buildTargetDir, "module.wasm") + embedWasm := filepath.Join(buildTargetDir, "embed.wasm") + componentWasm := filepath.Join(buildTargetDir, "component.wasm") + composedComponentWasm := filepath.Join(componentsTargetDir, fmt.Sprintf("%s.wasm", compName)) + + return serialRun( + func() error { return os.MkdirAll(buildTargetDir, 0755) }, + func() error { return os.MkdirAll(componentsTargetDir, 0755) }, + func() error { return GenerateBinding(witDir, bindingDir) }, + func() error { return TinyGoBuildComponentBinary(componentDir, moduleWasm) }, + func() error { return WASMToolsComponentEmbed(witDir, moduleWasm, embedWasm) }, + func() error { return WASMToolsComponentNew(embedWasm, componentWasm) }, + func() error { + return StubCompose(compName, componentWasm, composedComponentWasm) + }, + ) +} + +// GenerateBinding generates go binding from WIT +func GenerateBinding(witDir, bindingDir string) error { + return opRun(op{ + RunMessage: fmt.Sprintf("Generating bindings from %s into %s", witDir, bindingDir), + SkipMessage: "binding generation", + Targets: []string{bindingDir}, + SourcePaths: []string{witDir}, + Run: func() error { + return sh.RunV("wit-bindgen", "tiny-go", "--rename-package", "binding", "--out-dir", bindingDir, witDir) + }, + }) +} + +// TinyGoBuildComponentBinary build wasm component binary with tiny go +func TinyGoBuildComponentBinary(componentDir, moduleWasm string) error { + return opRun(op{ + RunMessage: fmt.Sprintf("Building component binary with tiny go: %s", moduleWasm), + SkipMessage: "tinygo component binary build", + Targets: []string{moduleWasm}, + SourcePaths: []string{componentsDir, libDir}, + Run: func() error { + return sh.RunV( + "tinygo", "build", "-target=wasi", "-tags=purego", + "-o", moduleWasm, + filepath.Join(componentDir, "main.go"), + ) + }, + }) +} + +// WASMToolsComponentEmbed embeds type info into wasm component with wasm-tools +func WASMToolsComponentEmbed(witDir, moduleWasm, embedWasm string) error { + return opRun(op{ + RunMessage: fmt.Sprintf("Embedding component type info (%s, %s) -> %s", moduleWasm, witDir, embedWasm), + SkipMessage: "wasm-tools component embed", + Targets: []string{embedWasm}, + SourcePaths: []string{witDir, moduleWasm}, + Run: func() error { + return sh.RunV( + "wasm-tools", "component", "embed", + witDir, moduleWasm, + "--output", embedWasm, + ) + }, + }) +} + +// WASMToolsComponentNew create golem component with wasm-tools +func WASMToolsComponentNew(embedWasm, componentWasm string) error { + return opRun(op{ + RunMessage: fmt.Sprintf("Creating new component: %s", embedWasm), + SkipMessage: "wasm-tools component new", + Targets: []string{componentWasm}, + SourcePaths: []string{embedWasm}, + Run: func() error { + return sh.RunV( + "wasm-tools", "component", "new", + embedWasm, + "-o", componentWasm, + "--adapt", wasiSnapshotPreview1Adapter, + ) + }, + }) +} + +// GenerateNewComponent generates a new component based on the component-template +func GenerateNewComponent(compName string) error { + err := sh.RunV("go", "run", "component-generator/main.go", pkgNs, compName) + if err != nil { + return fmt.Errorf("generate new component failed for %s, %w", compName, err) + } + + return nil +} + +// Clean cleans the projects +func Clean() error { + fmt.Println("Cleaning...") + + paths := []string{targetDir} + for _, compName := range compNames() { + paths = append(paths, filepath.Join(componentsDir, compName, "binding")) + } + + for _, path := range paths { + fmt.Printf("Deleting %s\n", path) + err := os.RemoveAll(path) + if err != nil { + return fmt.Errorf("clean: remove all failed for %s, %w", path, err) + } + } + + return nil +} + +// Deploy adds or updates all the components with golem-cli's default profile +func Deploy() error { + componentsTargetDir := filepath.Join(targetDir, "components") + for _, compName := range compNames() { + wasm := filepath.Join(componentsTargetDir, fmt.Sprintf("%s.wasm", compName)) + err := sh.RunV( + "golem-cli", "component", "add", + "--non-interactive", + "--component"+"-name", compName, + wasm, + ) + if err != nil { + return fmt.Errorf("deploy: failed for %s, %w", compName, err) + } + } + return nil +} + +// TestIntegration tests the deployed components +func TestIntegration() error { + err := sh.RunV("go", "test", "./integration", "-v") + if err != nil { + return fmt.Errorf("test integration failed: %w", err) + } + + return nil +} + +// compNames returns component names based on directories found in the components directory +func compNames() []string { + var compNames []string + dirs, err := os.ReadDir(componentsDir) + if err != nil { + return nil + } + for _, dir := range dirs { + compNames = append(compNames, dir.Name()) + } + return compNames +} + +// stubCompNames returns component names that need stubs based on the dependencies defined in componentDeps +func stubCompNames() []string { + compNamesSet := make(map[string]struct{}) + for _, deps := range componentDeps { + for _, dep := range deps { + compNamesSet[dep] = struct{}{} + } + } + + var compNames []string + for comp := range compNamesSet { + compNames = append(compNames, comp) + } + sort.Strings(compNames) + return compNames +} + +func copyFile(srcFileName, dstFileName string) error { + src, err := os.Open(srcFileName) + if err != nil { + return fmt.Errorf("copyFile: open failed for %s, %w", srcFileName, err) + } + defer func() { _ = src.Close() }() + + dst, err := os.Create(dstFileName) + if err != nil { + return fmt.Errorf("copyFile: create failed for %s, %w", srcFileName, err) + } + defer func() { _ = dst.Close() }() + + _, err = io.Copy(dst, src) + if err != nil { + return fmt.Errorf("copyFile: copy failed from %s to %s, %w", srcFileName, dstFileName, err) + } + + return nil +} + +func serialRun(fs ...func() error) error { + for i, f := range fs { + err := f() + if err != nil { + return fmt.Errorf("serialRun: step %d failed: %w", i+1, err) + } + } + return nil +} + +type op struct { + RunMessage string + SkipMessage string + Targets []string + SourcePaths []string + Run func() error +} + +func opRun(op op) error { + var run bool + if len(op.Targets) == 0 { + run = true + } else { + run = false + for _, t := range op.Targets { + var err error + run, err = target.Dir(t, op.SourcePaths...) + if err != nil { + return err + } + if run { + break + } + } + } + + if !run { + var targets string + if len(op.Targets) == 1 { + targets = op.Targets[0] + } else { + targets = fmt.Sprintf("(%s)", strings.Join(op.Targets, ", ")) + } + fmt.Printf("%s is up to date, skipping %s\n", targets, op.SkipMessage) + return nil + } + + fmt.Println(op.RunMessage) + return op.Run() +} diff --git a/examples/go/go-multi-rpc/metadata.json b/examples/go/go-multi-rpc/metadata.json new file mode 100644 index 0000000..be66d9d --- /dev/null +++ b/examples/go/go-multi-rpc/metadata.json @@ -0,0 +1,17 @@ +{ + "description": "Golem Go project with multiple components and worker-to-worker RPC calls, using magefile for managing the project", + "requiresAdapter": true, + "requiresGolemHostWIT": true, + "requiresWASI": true, + "witDepsPaths": [ + "component-template/component/wit/deps", + "components/component-one/wit/deps", + "components/component-two/wit/deps", + "components/component-three/wit/deps" + ], + "exclude": [ + "binding", + "target" + ], + "instructions": "INSTRUCTIONS-mage" +} \ No newline at end of file diff --git a/examples/go/go-multi-rpc/tools/tools.go b/examples/go/go-multi-rpc/tools/tools.go new file mode 100644 index 0000000..6d969b3 --- /dev/null +++ b/examples/go/go-multi-rpc/tools/tools.go @@ -0,0 +1,6 @@ +//go:build tools +// +build tools + +package tools + +import _ "github.com/magefile/mage/mg"