From e731306a1c19fa2f11c63ceda459a6900695065c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 27 Aug 2024 13:41:05 +0200 Subject: [PATCH 1/3] generator support for multi-component examples --- README.md | 8 ++++++-- src/lib.rs | 44 +++++++++++++++++++++++++++++++++----------- src/model.rs | 12 ++++++++++++ src/test/main.rs | 2 +- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7d2b762..0b98c49 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,11 @@ The following fields are required: The following fields are optional: - `requiresAdapter` is a boolean, defaults to **true**. If true, the appropriate version of the WASI Preview2 to Preview1 adapter is copied into the generated project (based on the guest language) to an `adapters` directory. -- `requiresGolemHostWIT` is a boolean, defaults to **false**. If true, the Golem specific WIT interface gets copied into `wit/deps`. +- `requiresGolemHostWIT` is a boolean, defaults to **false**. If true, the Golem specific WIT interface gets copied into `wit/deps`. - `requiresWASI` is a boolean, defaults to **false**. If true, the WASI Preview2 WIT interfaces which are compatible with Golem Cloud get copied into `wit/deps`. +- `witDepsPaths` is an array of directory paths, defaults to **null**. When set, overrides the `wit/deps` directory for the above options and allows to use multiple target dirs for supporting multi-component examples. - `exclude` is a list of sub-paths and works as a simplified `.gitignore` file. It's primary purpose is to help the development loop of working on examples and in the future it will likely be dropped in favor of just using `.gitignore` files. +- `instructions` is an optional filename, defaults to **null**. When set, overrides the __INSTRUCTIONS__ file used for the example, the file needs to be placed to same directory as the default instructions file. ### Template rules @@ -34,7 +36,7 @@ Golem examples are currently simple and not using any known template language, i When calling `golem-new` the user specifies a **template name**. The provided component name must use either `PascalCase`, `snake_case` or `kebab-case`. -There is an optional parameter for defining a **package name**, which defaults to `golem:component`. It has to be in the `pack:name` format. +There is an optional parameter for defining a **package name**, which defaults to `golem:component`. It has to be in the `pack:name` format. The first part of the package name is called **package namespace**. The following occurrences get replaced to the provided component name, applying the casing used in the template: - `component-name` @@ -46,6 +48,8 @@ The following occurrences get replaced to the provided component name, applying - `pack-name` - `pack/name` - `PackName` +- `pack-ns` +- `PackNs` ### Testing the examples The example generation and instructions can be tested with a test [cli app](/src/test/main.rs). diff --git a/src/lib.rs b/src/lib.rs index ffdd4af..13d68e8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,6 @@ impl Examples for GolemExamples { if let Some(lang_dir) = entry.as_dir() { let lang_dir_name = lang_dir.path().file_name().unwrap().to_str().unwrap(); if let Some(lang) = GuestLanguage::from_string(lang_dir_name) { - let instructions_path = lang_dir.path().join("INSTRUCTIONS"); let adapters_path = Path::new(lang.tier().name()).join("wasi_snapshot_preview1.wasm"); @@ -42,7 +41,8 @@ impl Examples for GolemExamples { { let example = parse_example( &lang, - &instructions_path, + &lang_dir.path(), + Path::new("INSTRUCTIONS"), &adapters_path, example_dir.path(), ); @@ -81,17 +81,29 @@ impl Examples for GolemExamples { .join(adapter_path.file_name().unwrap().to_str().unwrap()), )?; } - for wit_dep in &example.wit_deps { - copy_all( - &WIT, - wit_dep, - ¶meters + let wit_deps_targets = { + match &example.wit_deps_targets { + Some(paths) => paths + .iter() + .map(|path| { + parameters + .target_path + .join(parameters.component_name.as_string()) + .join(path) + }) + .collect(), + None => vec![parameters .target_path .join(parameters.component_name.as_string()) .join("wit") - .join("deps") - .join(wit_dep.file_name().unwrap().to_str().unwrap()), - )?; + .join("deps")], + } + }; + for wit_dep in &example.wit_deps { + for target_wit_deps in &wit_deps_targets { + let target = target_wit_deps.join(wit_dep.file_name().unwrap().to_str().unwrap()); + copy_all(&WIT, wit_dep, &target)?; + } } Ok(Self::instructions(example, parameters)) } @@ -205,6 +217,8 @@ fn transform(str: impl AsRef, parameters: &ExampleParameters) -> String { .replace("pack-name", ¶meters.package_name.to_kebab_case()) .replace("pack/name", ¶meters.package_name.to_string_with_slash()) .replace("PackName", ¶meters.package_name.to_pascal_case()) + .replace("pack-ns", ¶meters.package_name.namespace()) + .replace("PackNs", ¶meters.package_name.namespace_title_case()) } fn file_name_transform(str: impl AsRef, parameters: &ExampleParameters) -> String { @@ -213,7 +227,8 @@ fn file_name_transform(str: impl AsRef, parameters: &ExampleParameters) -> fn parse_example( lang: &GuestLanguage, - instructions_path: &Path, + lang_path: &Path, + default_instructions_file_name: &Path, adapters_path: &Path, example_root: &Path, ) -> Example { @@ -223,6 +238,10 @@ fn parse_example( .contents(); let metadata = serde_json::from_slice::(raw_metadata) .expect("Failed to parse metadata JSON"); + let instructions_path = match metadata.instructions { + Some(instructions_file_name) => lang_path.join(instructions_file_name), + None => lang_path.join(default_instructions_file_name), + }; let raw_instructions = EXAMPLES .get_file(instructions_path) .expect("Failed to read instructions") @@ -261,6 +280,9 @@ fn parse_example( None }, wit_deps, + wit_deps_targets: metadata + .wit_deps_paths + .map(|dirs| dirs.iter().map(|dir| PathBuf::from(dir)).collect()), exclude: metadata.exclude.iter().cloned().collect(), } } diff --git a/src/model.rs b/src/model.rs index 6651e64..e76aeac 100644 --- a/src/model.rs +++ b/src/model.rs @@ -257,6 +257,14 @@ impl PackageName { pub fn to_kebab_case(&self) -> String { format!("{}-{}", self.0 .0, self.0 .1) } + + pub fn namespace(&self) -> String { + self.0 .0.to_string() + } + + pub fn namespace_title_case(&self) -> String { + self.0 .0.to_title_case() + } } impl fmt::Display for PackageName { @@ -303,6 +311,7 @@ pub struct Example { pub instructions: String, pub adapter: Option, pub wit_deps: Vec, + pub wit_deps_targets: Option>, pub exclude: HashSet, } @@ -322,7 +331,10 @@ pub(crate) struct ExampleMetadata { pub requires_golem_host_wit: Option, #[serde(rename = "requiresWASI")] pub requires_wasi: Option, + #[serde(rename = "witDepsPaths")] + pub wit_deps_paths: Option>, pub exclude: Vec, + pub instructions: Option, } #[cfg(test)] diff --git a/src/test/main.rs b/src/test/main.rs index 3047387..9f3a2fc 100644 --- a/src/test/main.rs +++ b/src/test/main.rs @@ -84,7 +84,7 @@ fn test_example(command: &Command, example: &Example) -> Result<(), String> { ); let component_name = ComponentName::new(example.name.as_string().to_string() + "-comp"); let package_name = - PackageName::from_string("golem:component").ok_or("failed to create package name")?; + PackageName::from_string("golemx:componentx").ok_or("failed to create package name")?; let component_path = target_path.join(component_name.as_string()); println!("Target path: {}", target_path.display().to_string().blue()); From 4dd54a9186ad1bdb629d343175caa612315e89e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 27 Aug 2024 13:53:09 +0200 Subject: [PATCH 2/3] go-multi-rpc example --- examples/go/INSTRUCTIONS-mage | 14 + examples/go/go-multi-rpc/.gitignore | 2 + examples/go/go-multi-rpc/README.md | 188 ++++++++ .../go-multi-rpc/component-generator/main.go | 142 ++++++ .../component-template/component/main.go | 29 ++ .../component/wit/component.wit | 41 ++ .../components/component-one/main.go | 62 +++ .../component-one/wit/component-one.wit | 42 ++ .../pack-ns_component-three-stub/_stub.wit | 23 + .../component-three.wit | 41 ++ .../deps/pack-ns_component-two-stub/_stub.wit | 23 + .../pack-ns_component-two/component-two.wit | 41 ++ .../components/component-three/main.go | 29 ++ .../component-three/wit/component-three.wit | 41 ++ .../components/component-two/main.go | 49 ++ .../component-two/wit/component-two.wit | 41 ++ .../pack-ns_component-three-stub/_stub.wit | 23 + .../component-three.wit | 41 ++ examples/go/go-multi-rpc/go.mod | 15 + examples/go/go-multi-rpc/go.sum | 12 + .../integration/integration_test.go | 201 +++++++++ examples/go/go-multi-rpc/lib/cfg/cfg.go | 68 +++ examples/go/go-multi-rpc/mage.go | 14 + .../go/go-multi-rpc/magefiles/magefile.go | 423 ++++++++++++++++++ examples/go/go-multi-rpc/metadata.json | 17 + examples/go/go-multi-rpc/tools/tools.go | 6 + 26 files changed, 1628 insertions(+) create mode 100644 examples/go/INSTRUCTIONS-mage create mode 100644 examples/go/go-multi-rpc/.gitignore create mode 100644 examples/go/go-multi-rpc/README.md create mode 100644 examples/go/go-multi-rpc/component-generator/main.go create mode 100644 examples/go/go-multi-rpc/component-template/component/main.go create mode 100644 examples/go/go-multi-rpc/component-template/component/wit/component.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/main.go create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/component-one.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three-stub/_stub.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three/component-three.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two-stub/_stub.wit create mode 100644 examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two/component-two.wit create mode 100644 examples/go/go-multi-rpc/components/component-three/main.go create mode 100644 examples/go/go-multi-rpc/components/component-three/wit/component-three.wit create mode 100644 examples/go/go-multi-rpc/components/component-two/main.go create mode 100644 examples/go/go-multi-rpc/components/component-two/wit/component-two.wit create mode 100644 examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three-stub/_stub.wit create mode 100644 examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three/component-three.wit create mode 100644 examples/go/go-multi-rpc/go.mod create mode 100644 examples/go/go-multi-rpc/go.sum create mode 100644 examples/go/go-multi-rpc/integration/integration_test.go create mode 100644 examples/go/go-multi-rpc/lib/cfg/cfg.go create mode 100644 examples/go/go-multi-rpc/mage.go create mode 100644 examples/go/go-multi-rpc/magefiles/magefile.go create mode 100644 examples/go/go-multi-rpc/metadata.json create mode 100644 examples/go/go-multi-rpc/tools/tools.go diff --git a/examples/go/INSTRUCTIONS-mage b/examples/go/INSTRUCTIONS-mage new file mode 100644 index 0000000..22d4366 --- /dev/null +++ b/examples/go/INSTRUCTIONS-mage @@ -0,0 +1,14 @@ +See the documentation about installing tooling: https://learn.golem.cloud/docs/go-language-guide/setup +and see more details in the README.md file about other commands. + +Before first time build, generate RPC stubs: + go run mage.go updateRpcStubs + +Generate bindings, compile and compose components: + go run mage.go build + +Deploy components to default profile: + go run mage.go deploy + +Test the deployed components: + go run mage.go testIntegration diff --git a/examples/go/go-multi-rpc/.gitignore b/examples/go/go-multi-rpc/.gitignore new file mode 100644 index 0000000..c149d44 --- /dev/null +++ b/examples/go/go-multi-rpc/.gitignore @@ -0,0 +1,2 @@ +binding +/target/ diff --git a/examples/go/go-multi-rpc/README.md b/examples/go/go-multi-rpc/README.md new file mode 100644 index 0000000..2bbef76 --- /dev/null +++ b/examples/go/go-multi-rpc/README.md @@ -0,0 +1,188 @@ +# Golem Go Example with Multiple Components and Worker to Worker RPC Communication + +## Building +The project uses [magefile](https://magefile.org/) for building. Either install the tool binary `mage`, +or use the __zero install option__: `go run mage.go`. This readme will use the latter. + +To see the available commands use: + +```shell +go run mage.go +Targets: + addStubDependency adds generated and built stub dependency to componentGolemCliAddStubDependency + build alias for BuildAllComponents + buildAllComponents builds all components + buildComponent builds component by name + buildStubComponent builds RPC stub for component + clean cleans the projects + deploy adds or updates all the components with golem-cli\'s default profile + generateBinding generates go binding from WIT + generateNewComponent generates a new component based on the component-template + stubCompose composes dependencies + testIntegration tests the deployed components + tinyGoBuildComponentBinary build wasm component binary with tiny go + updateRpcStubs builds rpc stub components and adds them as dependency + wasmToolsComponentEmbed embeds type info into wasm component with wasm-tools + wasmToolsComponentNew create golem component with wasm-tools +``` + +For building the project for the first time (or after `clean`) use the following commands: + +```shell +go run mage.go updateRpcStubs +go run mage.go build +``` + +After this, using the `build` command is enough, unless there are changes in the RPC dependencies, +in that case `updateRpcStubs` is needed again. + +The final components that are usable by golem are placed in the `target/components` folder. + +## Deploying and testing the example + +In the example 3 simple counter components are defined, which can be familiar from the smaller examples. To showcase the remote calls, the counters `add` functions are connected, apart from increasing their own counter: + - **component one** delegates the add call to **component two** and **three** too, + - and **component two** delegates to **component three**. + +In both cases the _current worker name_ will be used as _target worker name_ too. + +Apart from _worker name_, remote calls also require the **target components' deployed ID**. For this the example uses environment variables, and uses the `lib/cfg` subpackage (which is shared between the components) to extract it. + +The examples assume a configured default `golem-cli` profile, and will use that. + +To test, first we have to build the project as seen in the above: + +```shell +go run mage.go updateRpcStubs +go run mage.go build +``` + +Then we can deploy our components with `golem-cli`, for which a wrapper magefile command is provided: + +```shell +go run mage.go deploy +``` + +Once the components are deployed, a simple example integration test suite can be used to test the components. +The tests are in the [/integration/integration_test.go](/integration/integration_test.go) test file, and can be run with: + +```shell +go run mage.go testIntegration +``` + +The `TestDeployed` simply tests if our components metadata is available through `golem-cli component get`. + +The `TestCallingAddOnComponentOneCallsToOtherComponents` will: + - get the _component URNs_ with `golem-cli component get` + - generates a _random worker name_, so our tests are starting from a clean state + - adds 1 - 1 worker for component one and component two with the required _environment variables_ containing the other workers' _component ids_ + - then makes various component invocations with `golem-cli worker invoke-and-await` and tests if the counters - after increments - are holding the right value according to the delegated `add` function calls. + +## Adding Components + +Use the `generateNewComponent` command to add new components to the project: + +```shell +go run mage.go generateNewComponent component-four +``` + +The above will create a new component in the `components/component-four` directory based on the template at [/component-template/component](/component-template/component). + +After adding a new component the `build` command will also include it. + +## Using Worker to Worker RPC calls + +### Under the hood + +Under the hood the _magefile_ commands below (and for build) use generic `golem-cli stubgen` subcommands: + - `golem-cli stubgen build` for creating remote call _stub WIT_ definitions and _WASM components_ for the stubs + - `golem-cli stubgen add-stub-dependency` for adding the _stub WIT_ definitions to a _component's WIT_ dependencies + - `golem-cli stubgen compose` for _composing_ components with the stub components + +### Magefile commands and required manual steps + +The dependencies between components are defined in the [/magefiles/magefile.go](/magefiles/magefile.go) build script: + +```go +// componentDeps defines the Worker to Worker RPC dependencies +var componentDeps = map[string][]string{ + "component-one": {"component-two", "component-three"}, + "component-two": {"component-three"}, +} +``` + +After changing dependencies the `updateRpcStubs` command can be used to create the necessary stubs: + +```shell +go run mage.go updateRpcStubs +``` + +The command will create stubs for the dependency projects in the ``/target/stub`` directory and will also place the required stub _WIT_ interfaces on the dependant component's `wit/deps` directory. + +To actually use the dependencies in a project it also has to be manually imported in the component's world. + +E.g. with the above definitions the following import has to be __manually__ added to `/components/component-one/wit/component-one.wit`: + +```wit +import pack-ns:component-two-stub; +import pack-ns:component-three-stub; +``` + +So the component definition should like similar to this: + +```wit +package pack-ns:component-one; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-one-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-one { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + // . + // . + // . + // other dependencies + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + import pack-ns:component-two-stub; + import pack-ns:component-three-stub; + + export component-one-api; +} +``` + +After this `build` (or the `generateBinding`) command can be used to update bindings, which now should include the +required functions for calling other components. + +Here's an example that delegates the `Add` call to another component and waits for the result: + +```go +import ( + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/component-one/binding" +) + + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + componentTwo := binding.NewComponentTwoApi(binding.GolemRpc0_1_0_TypesUri{Value: "uri"}) + defer componentTwo.Drop() + componentTwo.BlockingAdd(value) + + i.counter += value +} +``` + +Once a remote call is in place, the `build` command will also compose the stub components into the caller component. diff --git a/examples/go/go-multi-rpc/component-generator/main.go b/examples/go/go-multi-rpc/component-generator/main.go new file mode 100644 index 0000000..c5e7f4b --- /dev/null +++ b/examples/go/go-multi-rpc/component-generator/main.go @@ -0,0 +1,142 @@ +package main + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +func main() { + if len(os.Args) != 3 { + exit(0, fmt.Sprintf("Usage: %s ", os.Args[0])) + } + + componentTemplateRoot := "component-template/component" + pkgOrg := os.Args[1] + compName := os.Args[2] + componentDir := filepath.Join("components", compName) + + _, err := os.Stat(componentDir) + if err == nil { + exit(1, fmt.Sprintf("Target component directory already exists: %s", componentDir)) + } + if err != nil && !os.IsNotExist(err) { + exit(1, err.Error()) + } + + err = os.MkdirAll(componentDir, 0755) + if err != nil { + exit(1, err.Error()) + } + + err = fs.WalkDir( + os.DirFS(componentTemplateRoot), + ".", + func(path string, d fs.DirEntry, err error) error { + srcFilePath := filepath.Join(componentTemplateRoot, path) + fileInfo, err := os.Stat(srcFilePath) + if err != nil { + return fmt.Errorf("stat failed for template %s, %w", srcFilePath, err) + } + + if fileInfo.IsDir() { + return nil + } + + switch path { + case "main.go": + err = generateFile(pkgOrg, compName, srcFilePath, filepath.Join(componentDir, path)) + case "wit/component.wit": + err = generateFile(pkgOrg, compName, srcFilePath, filepath.Join(componentDir, "wit", compName+".wit")) + default: + err = copyFile(srcFilePath, filepath.Join(componentDir, path)) + } + if err != nil { + return fmt.Errorf("template generation failed for %s, %w", srcFilePath, err) + } + + return nil + }) + if err != nil { + exit(1, err.Error()) + } +} + +func generateFile(pkgOrg, compName, srcFileName, dstFileName string) error { + pascalPkgOrg := dashToPascal(pkgOrg) + pascalCompName := dashToPascal(compName) + + fmt.Printf("Generating from %s to %s\n", srcFileName, dstFileName) + + contentsBs, err := os.ReadFile(srcFileName) + if err != nil { + return fmt.Errorf("generateFile: read file failed for %s, %w", srcFileName, err) + } + + contents := string(contentsBs) + + contents = strings.ReplaceAll(contents, "comp-name", compName) + contents = strings.ReplaceAll(contents, "pck-ns", pkgOrg) + contents = strings.ReplaceAll(contents, "CompName", pascalCompName) + contents = strings.ReplaceAll(contents, "PckNs", pascalPkgOrg) + + dstDir := filepath.Dir(dstFileName) + err = os.MkdirAll(dstDir, 0755) + if err != nil { + return fmt.Errorf("generateFile: mkdir failed for %s, %w", dstDir, err) + } + + err = os.WriteFile(dstFileName, []byte(contents), 0644) + if err != nil { + return fmt.Errorf("generateFile: write file failed for %s, %w", dstFileName, err) + } + + return nil +} + +func dashToPascal(s string) string { + parts := strings.Split(s, "-") + for i, part := range parts { + if len(part) > 0 { + parts[i] = strings.ToUpper(string(part[0])) + part[1:] + } + } + return strings.Join(parts, "") +} + +func copyFile(srcFileName, dstFileName string) error { + fmt.Printf("Copy %s to %s\n", srcFileName, dstFileName) + + src, err := os.Open(srcFileName) + if err != nil { + return fmt.Errorf("copyFile: open failed for %s, %w", srcFileName, err) + } + defer func() { _ = src.Close() }() + + dstDir := filepath.Dir(dstFileName) + err = os.MkdirAll(dstDir, 0755) + if err != nil { + return fmt.Errorf("copyFile: mkdir failed for %s, %w", dstDir, err) + } + + dst, err := os.Create(dstFileName) + if err != nil { + return fmt.Errorf("copyFile: create file failed for %s, %w", dstFileName, err) + } + defer func() { _ = dst.Close() }() + + _, err = io.Copy(dst, src) + if err != nil { + return fmt.Errorf("copyFile: copy failed from %s to %s, %w", srcFileName, dstFileName, err) + } + + return nil +} + +func exit(code int, message string) { + fmt.Println(message) + os.Exit(code) +} diff --git a/examples/go/go-multi-rpc/component-template/component/main.go b/examples/go/go-multi-rpc/component-template/component/main.go new file mode 100644 index 0000000..99b55da --- /dev/null +++ b/examples/go/go-multi-rpc/component-template/component/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/comp-name/binding" +) + +func init() { + binding.SetExportsPckNsCompNameCompNameApi(&Impl{}) +} + +type Impl struct { + counter uint64 +} + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + i.counter += value +} + +func (i *Impl) Get() uint64 { + std.Init(std.Packages{Os: true, NetHttp: true}) + + return i.counter +} + +func main() {} diff --git a/examples/go/go-multi-rpc/component-template/component/wit/component.wit b/examples/go/go-multi-rpc/component-template/component/wit/component.wit new file mode 100644 index 0000000..1e7b6cf --- /dev/null +++ b/examples/go/go-multi-rpc/component-template/component/wit/component.wit @@ -0,0 +1,41 @@ +package pck-ns:comp-name; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface comp-name-api { + add: func(value: u64); + get: func() -> u64; +} + +world comp-name { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + // import pck-ns:name-stub/stub-name; + + export comp-name-api; +} diff --git a/examples/go/go-multi-rpc/components/component-one/main.go b/examples/go/go-multi-rpc/components/component-one/main.go new file mode 100644 index 0000000..02658fe --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/main.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + + "github.com/golemcloud/golem-go/golemhost" + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/component-one/binding" + // NOTE: use the lib folder to create common packages used by multiple components + "golem-go-project/lib/cfg" +) + +func init() { + binding.SetExportsPackNsComponentOneComponentOneApi(&Impl{}) +} + +type Impl struct { + counter uint64 +} + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + selfWorkerName := golemhost.GetSelfMetadata().WorkerId.WorkerName + + { + componentTwoWorkerURI, err := cfg.ComponentTwoWorkerURI(selfWorkerName) + if err != nil { + fmt.Printf("%+v\n", err) + return + } + + fmt.Printf("Calling %s...\n", componentTwoWorkerURI.Value) + componentTwo := binding.NewComponentTwoApi(binding.GolemRpc0_1_0_TypesUri(componentTwoWorkerURI)) + defer componentTwo.Drop() + componentTwo.BlockingAdd(value) + } + + { + componentThreeWorkerURI, err := cfg.ComponentThreeWorkerURI(selfWorkerName) + if err != nil { + fmt.Printf("%+v\n", err) + return + } + + fmt.Printf("Calling %s...\n", componentThreeWorkerURI.Value) + componentThree := binding.NewComponentThreeApi(binding.GolemRpc0_1_0_TypesUri(componentThreeWorkerURI)) + defer componentThree.Drop() + componentThree.BlockingAdd(value) + } + + i.counter += value +} + +func (i *Impl) Get() uint64 { + std.Init(std.Packages{Os: true, NetHttp: true}) + + return i.counter +} + +func main() {} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/component-one.wit b/examples/go/go-multi-rpc/components/component-one/wit/component-one.wit new file mode 100644 index 0000000..d9befa1 --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/component-one.wit @@ -0,0 +1,42 @@ +package pack-ns:component-one; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-one-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-one { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + import pack-ns:component-two-stub/stub-component-two; + import pack-ns:component-three-stub/stub-component-three; + + export component-one-api; +} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three-stub/_stub.wit b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three-stub/_stub.wit new file mode 100644 index 0000000..ab3de6b --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three-stub/_stub.wit @@ -0,0 +1,23 @@ +package pack-ns:component-three-stub; + +interface stub-component-three { + use golem:rpc/types@0.1.0.{uri as golem-rpc-uri}; + use wasi:io/poll@0.2.0.{pollable as wasi-io-pollable}; + + resource future-get-result { + subscribe: func() -> wasi-io-pollable; + get: func() -> option; + } + resource component-three-api { + constructor(location: golem-rpc-uri); + blocking-add: func(value: u64); + add: func(value: u64); + blocking-get: func() -> u64; + get: func() -> future-get-result; + } + +} + +world wasm-rpc-stub-component-three { + export stub-component-three; +} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three/component-three.wit b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three/component-three.wit new file mode 100644 index 0000000..7bd3d9c --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-three/component-three.wit @@ -0,0 +1,41 @@ +package pack-ns:component-three; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-three-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-three { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + // e.g: import pack-ns:name-stub/stub-name; + + export component-three-api; +} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two-stub/_stub.wit b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two-stub/_stub.wit new file mode 100644 index 0000000..f83aa85 --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two-stub/_stub.wit @@ -0,0 +1,23 @@ +package pack-ns:component-two-stub; + +interface stub-component-two { + use golem:rpc/types@0.1.0.{uri as golem-rpc-uri}; + use wasi:io/poll@0.2.0.{pollable as wasi-io-pollable}; + + resource future-get-result { + subscribe: func() -> wasi-io-pollable; + get: func() -> option; + } + resource component-two-api { + constructor(location: golem-rpc-uri); + blocking-add: func(value: u64); + add: func(value: u64); + blocking-get: func() -> u64; + get: func() -> future-get-result; + } + +} + +world wasm-rpc-stub-component-two { + export stub-component-two; +} diff --git a/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two/component-two.wit b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two/component-two.wit new file mode 100644 index 0000000..bb81e6d --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-one/wit/deps/pack-ns_component-two/component-two.wit @@ -0,0 +1,41 @@ +package pack-ns:component-two; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-two-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-two { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + //import pack-ns:component-three-stub/stub-component-three; + + export component-two-api; +} diff --git a/examples/go/go-multi-rpc/components/component-three/main.go b/examples/go/go-multi-rpc/components/component-three/main.go new file mode 100644 index 0000000..646e5f1 --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-three/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/component-three/binding" +) + +func init() { + binding.SetExportsPackNsComponentThreeComponentThreeApi(&Impl{}) +} + +type Impl struct { + counter uint64 +} + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + i.counter += value +} + +func (i *Impl) Get() uint64 { + std.Init(std.Packages{Os: true, NetHttp: true}) + + return i.counter +} + +func main() {} diff --git a/examples/go/go-multi-rpc/components/component-three/wit/component-three.wit b/examples/go/go-multi-rpc/components/component-three/wit/component-three.wit new file mode 100644 index 0000000..7bd3d9c --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-three/wit/component-three.wit @@ -0,0 +1,41 @@ +package pack-ns:component-three; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-three-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-three { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + // e.g: import pack-ns:name-stub/stub-name; + + export component-three-api; +} diff --git a/examples/go/go-multi-rpc/components/component-two/main.go b/examples/go/go-multi-rpc/components/component-two/main.go new file mode 100644 index 0000000..8614a27 --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-two/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + + "github.com/golemcloud/golem-go/golemhost" + "github.com/golemcloud/golem-go/std" + + "golem-go-project/components/component-two/binding" + // NOTE: use the lib folder to create common packages used by multiple components + "golem-go-project/lib/cfg" +) + +func init() { + binding.SetExportsPackNsComponentTwoComponentTwoApi(&Impl{}) +} + +type Impl struct { + counter uint64 +} + +func (i *Impl) Add(value uint64) { + std.Init(std.Packages{Os: true, NetHttp: true}) + + selfWorkerName := golemhost.GetSelfMetadata().WorkerId.WorkerName + + { + componentThreeWorkerURI, err := cfg.ComponentThreeWorkerURI(selfWorkerName) + if err != nil { + fmt.Printf("%+v\n", err) + return + } + + fmt.Printf("Calling %s...\n", componentThreeWorkerURI.Value) + componentThree := binding.NewComponentThreeApi(binding.GolemRpc0_1_0_TypesUri(componentThreeWorkerURI)) + defer componentThree.Drop() + componentThree.BlockingAdd(value) + } + + i.counter += value +} + +func (i *Impl) Get() uint64 { + std.Init(std.Packages{Os: true, NetHttp: true}) + + return i.counter +} + +func main() {} diff --git a/examples/go/go-multi-rpc/components/component-two/wit/component-two.wit b/examples/go/go-multi-rpc/components/component-two/wit/component-two.wit new file mode 100644 index 0000000..6ed354b --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-two/wit/component-two.wit @@ -0,0 +1,41 @@ +package pack-ns:component-two; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-two-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-two { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + import pack-ns:component-three-stub/stub-component-three; + + export component-two-api; +} diff --git a/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three-stub/_stub.wit b/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three-stub/_stub.wit new file mode 100644 index 0000000..ab3de6b --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three-stub/_stub.wit @@ -0,0 +1,23 @@ +package pack-ns:component-three-stub; + +interface stub-component-three { + use golem:rpc/types@0.1.0.{uri as golem-rpc-uri}; + use wasi:io/poll@0.2.0.{pollable as wasi-io-pollable}; + + resource future-get-result { + subscribe: func() -> wasi-io-pollable; + get: func() -> option; + } + resource component-three-api { + constructor(location: golem-rpc-uri); + blocking-add: func(value: u64); + add: func(value: u64); + blocking-get: func() -> u64; + get: func() -> future-get-result; + } + +} + +world wasm-rpc-stub-component-three { + export stub-component-three; +} diff --git a/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three/component-three.wit b/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three/component-three.wit new file mode 100644 index 0000000..7bd3d9c --- /dev/null +++ b/examples/go/go-multi-rpc/components/component-two/wit/deps/pack-ns_component-three/component-three.wit @@ -0,0 +1,41 @@ +package pack-ns:component-three; + +// See https://component-model.bytecodealliance.org/design/wit.html for more details about the WIT syntax + +interface component-three-api { + add: func(value: u64); + get: func() -> u64; +} + +world component-three { + // Golem dependencies + import golem:api/host@0.2.0; + import golem:rpc/types@0.1.0; + + // WASI dependencies + import wasi:blobstore/blobstore; + import wasi:blobstore/container; + import wasi:cli/environment@0.2.0; + import wasi:clocks/wall-clock@0.2.0; + import wasi:clocks/monotonic-clock@0.2.0; + import wasi:filesystem/preopens@0.2.0; + import wasi:filesystem/types@0.2.0; + import wasi:http/types@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + import wasi:io/error@0.2.0; + import wasi:io/poll@0.2.0; + import wasi:io/streams@0.2.0; + // import wasi:keyvalue/eventual-batch@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + // import wasi:keyvalue/eventual@0.1.0; // NOTE: cannot include it because the bindings collide with blobstore + import wasi:logging/logging; + import wasi:random/random@0.2.0; + import wasi:random/insecure@0.2.0; + import wasi:random/insecure-seed@0.2.0; + import wasi:sockets/ip-name-lookup@0.2.0; + import wasi:sockets/instance-network@0.2.0; + + // Project Component dependencies + // e.g: import pack-ns:name-stub/stub-name; + + export component-three-api; +} diff --git a/examples/go/go-multi-rpc/go.mod b/examples/go/go-multi-rpc/go.mod new file mode 100644 index 0000000..5040c43 --- /dev/null +++ b/examples/go/go-multi-rpc/go.mod @@ -0,0 +1,15 @@ +module golem-go-project + +go 1.20.0 + +require ( + github.com/golemcloud/golem-go v0.7.0 + github.com/google/uuid v1.6.0 + github.com/magefile/mage v1.15.0 + github.com/tidwall/gjson v1.17.3 +) + +require ( + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect +) diff --git a/examples/go/go-multi-rpc/go.sum b/examples/go/go-multi-rpc/go.sum new file mode 100644 index 0000000..95966ad --- /dev/null +++ b/examples/go/go-multi-rpc/go.sum @@ -0,0 +1,12 @@ +github.com/golemcloud/golem-go v0.7.0 h1:8zpNeAtEDJsJllqHUmjji/lyplSJ0kI3v7vN97/N8zs= +github.com/golemcloud/golem-go v0.7.0/go.mod h1:VLL22qVo5R2+jGLO43tLPpPjf2WrA6B7GQoqXKnSODo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/examples/go/go-multi-rpc/integration/integration_test.go b/examples/go/go-multi-rpc/integration/integration_test.go new file mode 100644 index 0000000..1be0d0a --- /dev/null +++ b/examples/go/go-multi-rpc/integration/integration_test.go @@ -0,0 +1,201 @@ +package integration + +import ( + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/magefile/mage/sh" + "github.com/tidwall/gjson" +) + +func TestDeployed(t *testing.T) { + componentOneURN := mustGetCompURNByCompName(t, "component-one") + fmt.Printf("component-one: %s\n", componentOneURN) + + componentTwoURN := mustGetCompURNByCompName(t, "component-two") + fmt.Printf("component-two: %s\n", componentTwoURN) + + componentThreeURN := mustGetCompURNByCompName(t, "component-three") + fmt.Printf("component-three: %s\n", componentThreeURN) +} + +func TestCallingAddOnComponentOneCallsToOtherComponents(t *testing.T) { + workerName := uuid.New().String() + fmt.Printf("random worker name for test: %s\n", workerName) + + // Setup + { + // Get all component URNs + componentURNs := mustGetComponentURNs(t) + fmt.Printf("component urns: %#v\n", componentURNs) + + // Preparing workers with env vars for RPC, so they know the other component IDs + mustAddComponent(t, "component-one", workerName, componentURNs) + mustAddComponent(t, "component-two", workerName, componentURNs) + } + + // Call get on all component and check the counters are 0 + { + expectCounter(t, "component-one", workerName, 0) + expectCounter(t, "component-two", workerName, 0) + expectCounter(t, "component-three", workerName, 0) + } + + // Invoke add on component-one + { + mustInvokeAndAwaitComponent(t, "component-one", workerName, "pack-ns:component-one/component-one-api.{add}", []string{"3"}) + } + + // Call get on all component and check the counters are accumulated on component two and three + { + expectCounter(t, "component-one", workerName, 3) + expectCounter(t, "component-two", workerName, 3) + expectCounter(t, "component-three", workerName, 6) + } + + // Invoke add on component-two + { + mustInvokeAndAwaitComponent(t, "component-two", workerName, "pack-ns:component-two/component-two-api.{add}", []string{"2"}) + } + + // Call get on all component and check the counters are accumulated on component three + { + expectCounter(t, "component-one", workerName, 3) + expectCounter(t, "component-two", workerName, 5) + expectCounter(t, "component-three", workerName, 8) + } + + // Invoke add on component-one again + { + mustInvokeAndAwaitComponent(t, "component-one", workerName, "pack-ns:component-one/component-one-api.{add}", []string{"1"}) + } + + // Call get on all component and check the counters are accumulated on component two and three + { + expectCounter(t, "component-one", workerName, 4) + expectCounter(t, "component-two", workerName, 6) + expectCounter(t, "component-three", workerName, 10) + } +} + +func getCompURNByCompName(compName string) (string, error) { + output, err := sh.Output( + "golem-cli", "--format", "json", "component", "get", "--component"+"-name", compName, + ) + if err != nil { + return "", fmt.Errorf("getCompURNByCompName for %s: golem-cli failed: %w\n", compName, err) + } + + componentURN := gjson.Get(output, "componentUrn").String() + if componentURN == "" { + return "", fmt.Errorf("missing componentURN in response:\n%s\n", output) + } + + return componentURN, nil +} + +func mustGetCompURNByCompName(t *testing.T, compName string) string { + name, err := getCompURNByCompName(compName) + if err != nil { + t.Fatalf("%+v", err) + } + return name +} + +type ComponentURNs struct { + ComponentOne string + ComponentTwo string + ComponentThree string +} + +func mustGetComponentURNs(t *testing.T) ComponentURNs { + return ComponentURNs{ + ComponentOne: mustGetCompURNByCompName(t, "component-one"), + ComponentTwo: mustGetCompURNByCompName(t, "component-two"), + ComponentThree: mustGetCompURNByCompName(t, "component-three"), + } +} + +func addComponent(compName, workerName string, componentURNs ComponentURNs) error { + fmt.Printf("adding component: %s, %s\n", compName, workerName) + output, err := sh.Output( + "golem-cli", "worker", + "--format", "json", + "add", + "--component"+"-name", compName, + "--worker-name", workerName, + "--env", fmt.Sprintf("COMPONENT_ONE_ID=%s", componentIDFromURN(componentURNs.ComponentOne)), + "--env", fmt.Sprintf("COMPONENT_TWO_ID=%s", componentIDFromURN(componentURNs.ComponentTwo)), + "--env", fmt.Sprintf("COMPONENT_THREE_ID=%s", componentIDFromURN(componentURNs.ComponentThree)), + ) + if err != nil { + return fmt.Errorf("addComponent for %s, %s: golem-cli failed: %w\n%s", compName, workerName, err, output) + } + return nil +} + +func mustAddComponent(t *testing.T, compName, workerName string, componentURNs ComponentURNs) { + err := addComponent(compName, workerName, componentURNs) + if err != nil { + t.Fatalf("%+v", err) + } +} + +func invokeAndAwaitComponent(compName, workerName, function string, functionArgs []string) (string, error) { + fmt.Printf("invoking component: %s, %s, %s, %+v\n", compName, workerName, function, functionArgs) + + cliArgs := []string{ + "--format", "json", + "worker", + "invoke-and-await", + "--component" + "-name", compName, + "--worker-name", workerName, + "--function", function, + } + + for _, arg := range functionArgs { + cliArgs = append(cliArgs, []string{"--arg", arg}...) + } + + output, err := sh.Output("golem-cli", cliArgs...) + if err != nil { + return "", fmt.Errorf("invokeAndAwaitComponent failed: %w", err) + } + + fmt.Println(output) + + return output, nil +} + +func mustInvokeAndAwaitComponent(t *testing.T, componentURN, workerName, function string, functionArgs []string) string { + output, err := invokeAndAwaitComponent(componentURN, workerName, function, functionArgs) + if err != nil { + t.Fatalf("%+v", err) + } + return output +} + +func componentIDFromURN(urn string) string { + return strings.Split(urn, ":")[2] +} + +func expectCounter(t *testing.T, compName, workerName string, expected int64) { + output := mustInvokeAndAwaitComponent(t, compName, workerName, fmt.Sprintf("pack-ns:%s/%s-api.{get}", compName, compName), nil) + + actualValue := gjson.Get(output, "value") + if !actualValue.Exists() { + t.Fatalf("Expected counter for %s, %s: %d, actual value is missing", compName, workerName, expected) + } + + actualArray := actualValue.Array() + if len(actualArray) != 1 { + t.Fatalf("Expected counter for %s, %s: %d, actual value tuple has bad number of elements: %s", compName, workerName, expected, actualValue) + } + + actual := actualArray[0].Int() + if expected != actual { + t.Fatalf("Expected counter for %s, %s: %d, actual: %d", compName, workerName, expected, actual) + } +} diff --git a/examples/go/go-multi-rpc/lib/cfg/cfg.go b/examples/go/go-multi-rpc/lib/cfg/cfg.go new file mode 100644 index 0000000..f6bc382 --- /dev/null +++ b/examples/go/go-multi-rpc/lib/cfg/cfg.go @@ -0,0 +1,68 @@ +package cfg + +import ( + "fmt" + "os" + "strings" + + "github.com/google/uuid" + + "github.com/golemcloud/golem-go/binding" + "github.com/golemcloud/golem-go/golemhost" +) + +func ComponentIDFromEnv(key string) (golemhost.ComponentID, error) { + value := os.Getenv(key) + if value == "" { + return [16]byte{}, fmt.Errorf("missing environment variable for component id: %s", key) + } + componentID, err := uuid.Parse(strings.ToLower(value)) + if err != nil { + return [16]byte{}, fmt.Errorf("component id parse failed for %s=%s, %w", key, value, err) + } + return golemhost.ComponentID(componentID), nil +} + +func ComponentOneID() (golemhost.ComponentID, error) { + return ComponentIDFromEnv("COMPONENT_ONE_ID") +} + +func ComponentTwoID() (golemhost.ComponentID, error) { + return ComponentIDFromEnv("COMPONENT_TWO_ID") +} + +func ComponentThreeID() (golemhost.ComponentID, error) { + return ComponentIDFromEnv("COMPONENT_THREE_ID") +} + +func ComponentOneWorkerURI(workerName string) (binding.GolemRpc0_1_0_TypesUri, error) { + uri, err := workerURIF(ComponentOneID, workerName) + return uri, err +} + +func ComponentTwoWorkerURI(workerName string) (binding.GolemRpc0_1_0_TypesUri, error) { + uri, err := workerURIF(ComponentTwoID, workerName) + return uri, err +} + +func ComponentThreeWorkerURI(workerName string) (binding.GolemRpc0_1_0_TypesUri, error) { + uri, err := workerURIF(ComponentThreeID, workerName) + return uri, err +} + +func WorkerURI[T binding.GolemRpc0_1_0_TypesUri](workerID golemhost.WorkerID) T { + return T(binding.GolemRpc0_1_0_TypesUri{ + Value: fmt.Sprintf("urn:worker:%s/%s", (uuid.UUID(workerID.ComponentID)).URN(), workerID.WorkerName), + }) +} + +func workerURIF(getComponentID func() (golemhost.ComponentID, error), workerName string) (binding.GolemRpc0_1_0_TypesUri, error) { + componentID, err := getComponentID() + if err != nil { + return binding.GolemRpc0_1_0_TypesUri{}, err + } + return WorkerURI(golemhost.WorkerID{ + ComponentID: componentID, + WorkerName: workerName, + }), nil +} diff --git a/examples/go/go-multi-rpc/mage.go b/examples/go/go-multi-rpc/mage.go new file mode 100644 index 0000000..53c564a --- /dev/null +++ b/examples/go/go-multi-rpc/mage.go @@ -0,0 +1,14 @@ +//go:build ignore +// +build ignore + +package main + +import ( + "os" + + "github.com/magefile/mage/mage" +) + +func main() { + os.Exit(mage.Main()) +} diff --git a/examples/go/go-multi-rpc/magefiles/magefile.go b/examples/go/go-multi-rpc/magefiles/magefile.go new file mode 100644 index 0000000..61ac112 --- /dev/null +++ b/examples/go/go-multi-rpc/magefiles/magefile.go @@ -0,0 +1,423 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/magefile/mage/sh" + "github.com/magefile/mage/target" +) + +// componentDeps defines the Worker to Worker RPC dependencies +var componentDeps = map[string][]string{ + "component-one": {"component-two", "component-three"}, + "component-two": {"component-three"}, +} + +var pkgNs = "pack-ns" +var targetDir = "target" +var componentsDir = "components" +var libDir = "lib" +var wasiSnapshotPreview1Adapter = "adapters/tier1/wasi_snapshot_preview1.wasm" + +// Build alias for BuildAllComponents +func Build() error { + return BuildAllComponents() +} + +// BuildAllComponents builds all components +func BuildAllComponents() error { + for _, compName := range compNames() { + err := BuildComponent(compName) + if err != nil { + return fmt.Errorf("build all components: build component failed for %s, %w", compName, err) + } + } + + return nil +} + +// UpdateRpcStubs builds rpc stub components and adds them as dependency +func UpdateRpcStubs() error { + for _, compName := range stubCompNames() { + err := BuildStubComponent(compName) + if err != nil { + return fmt.Errorf("update RPC stubs: build stub component failed for %s, %w", compName, err) + } + } + + for _, compName := range compNames() { + for _, dependency := range componentDeps[compName] { + err := AddStubDependency(compName, dependency) + if err != nil { + return fmt.Errorf("update RPC stubs: add stub dependecy failed for %s to %s, %w", dependency, compName, err) + } + } + } + + return nil +} + +// BuildStubComponent builds RPC stub for component +func BuildStubComponent(compName string) error { + componentDir := filepath.Join(componentsDir, compName) + srcWitDir := filepath.Join(componentDir, "wit") + stubTargetDir := filepath.Join(targetDir, "stub", compName) + destWasm := filepath.Join(stubTargetDir, "stub.wasm") + destWitDir := filepath.Join(stubTargetDir, "wit") + + return opRun(op{ + RunMessage: fmt.Sprintf("Building stub component for %s", compName), + SkipMessage: "stub component build", + Targets: []string{destWasm, destWitDir}, + SourcePaths: []string{srcWitDir}, + Run: func() error { + return sh.RunV( + "golem-cli", "stubgen", "build", + "--source-wit-root", srcWitDir, + "--dest-wasm", destWasm, + "--dest-wit-root", destWitDir, + ) + }, + }) +} + +// AddStubDependency adds generated and built stub dependency to componentGolemCliAddStubDependency +func AddStubDependency(compName, depCompName string) error { + stubTargetDir := filepath.Join(targetDir, "stub", depCompName) + srcWitDir := filepath.Join(stubTargetDir, "wit") + dstComponentDir := filepath.Join(componentsDir, compName) + dstWitDir := filepath.Join(dstComponentDir, "wit") + dstWitDepDir := filepath.Join(dstComponentDir, dstWitDir, "deps", fmt.Sprintf("%s_%s", pkgNs, compName)) + dstWitDepStubDir := filepath.Join(dstComponentDir, dstWitDir, "deps", fmt.Sprintf("%s_%s-stub", pkgNs, compName)) + + return opRun(op{ + RunMessage: fmt.Sprintf("Adding stub dependecy for %s to %s", depCompName, compName), + SkipMessage: "add stub dependency", + Targets: []string{dstWitDepDir, dstWitDepStubDir}, + SourcePaths: []string{srcWitDir}, + Run: func() error { + return sh.RunV( + "golem-cli", "stubgen", "add-stub-dependency", + "--overwrite", + "--stub-wit-root", srcWitDir, + "--dest-wit-root", dstWitDir, + ) + }, + }) +} + +// StubCompose composes dependencies +func StubCompose(compName, componentWasm, targetWasm string) error { + buildTargetDir := filepath.Dir(componentWasm) + dependencies := componentDeps[compName] + + stubWasms := make([]string, len(dependencies)) + for i, compName := range dependencies { + stubTargetDir := filepath.Join(targetDir, "stub", compName) + stubWasms[i] = filepath.Join(stubTargetDir, "stub.wasm") + } + + return opRun(op{ + RunMessage: fmt.Sprintf("Composing %s into %s", fmt.Sprintf("[%s]", strings.Join(stubWasms, ", ")), compName), + SkipMessage: "composing", + Targets: []string{targetWasm}, + SourcePaths: append(stubWasms, componentWasm), + Run: func() error { + composeWasm := componentWasm + if len(stubWasms) > 0 { + srcWasm := componentWasm + for i, stubWasm := range stubWasms { + prevComposeWasm := composeWasm + composeWasm = filepath.Join( + buildTargetDir, + fmt.Sprintf("compose-%d-%s.wasm", i+1, filepath.Base(dependencies[i])), + ) + + outBuf := &bytes.Buffer{} + errBuff := &bytes.Buffer{} + + _, err := sh.Exec( + nil, outBuf, errBuff, + "golem-cli", "stubgen", "compose", + "--source-wasm", srcWasm, + "--stub-wasm", stubWasm, + "--dest-wasm", composeWasm, + ) + if err != nil { + errString := errBuff.String() + if strings.Contains(errString, "Error: no dependencies of component") && + strings.Contains(errString, "were found") { + fmt.Printf("Skipping composing %s, not used\n", stubWasm) + composeWasm = prevComposeWasm + continue + } + + fmt.Print(outBuf) + fmt.Print(errBuff) + + return fmt.Errorf("StubCompose failed: %w", err) + } + srcWasm = composeWasm + } + } + + return copyFile(composeWasm, targetWasm) + }, + }) +} + +// BuildComponent builds component by name +func BuildComponent(compName string) error { + componentDir := filepath.Join(componentsDir, compName) + witDir := filepath.Join(componentDir, "wit") + bindingDir := filepath.Join(componentDir, "binding") + buildTargetDir := filepath.Join(targetDir, "build", compName) + componentsTargetDir := filepath.Join(targetDir, "components") + moduleWasm := filepath.Join(buildTargetDir, "module.wasm") + embedWasm := filepath.Join(buildTargetDir, "embed.wasm") + componentWasm := filepath.Join(buildTargetDir, "component.wasm") + composedComponentWasm := filepath.Join(componentsTargetDir, fmt.Sprintf("%s.wasm", compName)) + + return serialRun( + func() error { return os.MkdirAll(buildTargetDir, 0755) }, + func() error { return os.MkdirAll(componentsTargetDir, 0755) }, + func() error { return GenerateBinding(witDir, bindingDir) }, + func() error { return TinyGoBuildComponentBinary(componentDir, moduleWasm) }, + func() error { return WASMToolsComponentEmbed(witDir, moduleWasm, embedWasm) }, + func() error { return WASMToolsComponentNew(embedWasm, componentWasm) }, + func() error { + return StubCompose(compName, componentWasm, composedComponentWasm) + }, + ) +} + +// GenerateBinding generates go binding from WIT +func GenerateBinding(witDir, bindingDir string) error { + return opRun(op{ + RunMessage: fmt.Sprintf("Generating bindings from %s into %s", witDir, bindingDir), + SkipMessage: "binding generation", + Targets: []string{bindingDir}, + SourcePaths: []string{witDir}, + Run: func() error { + return sh.RunV("wit-bindgen", "tiny-go", "--rename-package", "binding", "--out-dir", bindingDir, witDir) + }, + }) +} + +// TinyGoBuildComponentBinary build wasm component binary with tiny go +func TinyGoBuildComponentBinary(componentDir, moduleWasm string) error { + return opRun(op{ + RunMessage: fmt.Sprintf("Building component binary with tiny go: %s", moduleWasm), + SkipMessage: "tinygo component binary build", + Targets: []string{moduleWasm}, + SourcePaths: []string{componentsDir, libDir}, + Run: func() error { + return sh.RunV( + "tinygo", "build", "-target=wasi", "-tags=purego", + "-o", moduleWasm, + filepath.Join(componentDir, "main.go"), + ) + }, + }) +} + +// WASMToolsComponentEmbed embeds type info into wasm component with wasm-tools +func WASMToolsComponentEmbed(witDir, moduleWasm, embedWasm string) error { + return opRun(op{ + RunMessage: fmt.Sprintf("Embedding component type info (%s, %s) -> %s", moduleWasm, witDir, embedWasm), + SkipMessage: "wasm-tools component embed", + Targets: []string{embedWasm}, + SourcePaths: []string{witDir, moduleWasm}, + Run: func() error { + return sh.RunV( + "wasm-tools", "component", "embed", + witDir, moduleWasm, + "--output", embedWasm, + ) + }, + }) +} + +// WASMToolsComponentNew create golem component with wasm-tools +func WASMToolsComponentNew(embedWasm, componentWasm string) error { + return opRun(op{ + RunMessage: fmt.Sprintf("Creating new component: %s", embedWasm), + SkipMessage: "wasm-tools component new", + Targets: []string{componentWasm}, + SourcePaths: []string{embedWasm}, + Run: func() error { + return sh.RunV( + "wasm-tools", "component", "new", + embedWasm, + "-o", componentWasm, + "--adapt", wasiSnapshotPreview1Adapter, + ) + }, + }) +} + +// GenerateNewComponent generates a new component based on the component-template +func GenerateNewComponent(compName string) error { + err := sh.RunV("go", "run", "component-generator/main.go", pkgNs, compName) + if err != nil { + return fmt.Errorf("generate new component failed for %s, %w", compName, err) + } + + return nil +} + +// Clean cleans the projects +func Clean() error { + fmt.Println("Cleaning...") + + paths := []string{targetDir} + for _, compName := range compNames() { + paths = append(paths, filepath.Join(componentsDir, compName, "binding")) + } + + for _, path := range paths { + fmt.Printf("Deleting %s\n", path) + err := os.RemoveAll(path) + if err != nil { + return fmt.Errorf("clean: remove all failed for %s, %w", path, err) + } + } + + return nil +} + +// Deploy adds or updates all the components with golem-cli's default profile +func Deploy() error { + componentsTargetDir := filepath.Join(targetDir, "components") + for _, compName := range compNames() { + wasm := filepath.Join(componentsTargetDir, fmt.Sprintf("%s.wasm", compName)) + err := sh.RunV( + "golem-cli", "component", "add", + "--non-interactive", + "--component"+"-name", compName, + wasm, + ) + if err != nil { + return fmt.Errorf("deploy: failed for %s, %w", compName, err) + } + } + return nil +} + +// TestIntegration tests the deployed components +func TestIntegration() error { + err := sh.RunV("go", "test", "./integration", "-v") + if err != nil { + return fmt.Errorf("test integration failed: %w", err) + } + + return nil +} + +// compNames returns component names based on directories found in the components directory +func compNames() []string { + var compNames []string + dirs, err := os.ReadDir(componentsDir) + if err != nil { + return nil + } + for _, dir := range dirs { + compNames = append(compNames, dir.Name()) + } + return compNames +} + +// stubCompNames returns component names that need stubs based on the dependencies defined in componentDeps +func stubCompNames() []string { + compNamesSet := make(map[string]struct{}) + for _, deps := range componentDeps { + for _, dep := range deps { + compNamesSet[dep] = struct{}{} + } + } + + var compNames []string + for comp := range compNamesSet { + compNames = append(compNames, comp) + } + sort.Strings(compNames) + return compNames +} + +func copyFile(srcFileName, dstFileName string) error { + src, err := os.Open(srcFileName) + if err != nil { + return fmt.Errorf("copyFile: open failed for %s, %w", srcFileName, err) + } + defer func() { _ = src.Close() }() + + dst, err := os.Create(dstFileName) + if err != nil { + return fmt.Errorf("copyFile: create failed for %s, %w", srcFileName, err) + } + defer func() { _ = dst.Close() }() + + _, err = io.Copy(dst, src) + if err != nil { + return fmt.Errorf("copyFile: copy failed from %s to %s, %w", srcFileName, dstFileName, err) + } + + return nil +} + +func serialRun(fs ...func() error) error { + for i, f := range fs { + err := f() + if err != nil { + return fmt.Errorf("serialRun: step %d failed: %w", i+1, err) + } + } + return nil +} + +type op struct { + RunMessage string + SkipMessage string + Targets []string + SourcePaths []string + Run func() error +} + +func opRun(op op) error { + var run bool + if len(op.Targets) == 0 { + run = true + } else { + run = false + for _, t := range op.Targets { + var err error + run, err = target.Dir(t, op.SourcePaths...) + if err != nil { + return err + } + if run { + break + } + } + } + + if !run { + var targets string + if len(op.Targets) == 1 { + targets = op.Targets[0] + } else { + targets = fmt.Sprintf("(%s)", strings.Join(op.Targets, ", ")) + } + fmt.Printf("%s is up to date, skipping %s\n", targets, op.SkipMessage) + return nil + } + + fmt.Println(op.RunMessage) + return op.Run() +} diff --git a/examples/go/go-multi-rpc/metadata.json b/examples/go/go-multi-rpc/metadata.json new file mode 100644 index 0000000..be66d9d --- /dev/null +++ b/examples/go/go-multi-rpc/metadata.json @@ -0,0 +1,17 @@ +{ + "description": "Golem Go project with multiple components and worker-to-worker RPC calls, using magefile for managing the project", + "requiresAdapter": true, + "requiresGolemHostWIT": true, + "requiresWASI": true, + "witDepsPaths": [ + "component-template/component/wit/deps", + "components/component-one/wit/deps", + "components/component-two/wit/deps", + "components/component-three/wit/deps" + ], + "exclude": [ + "binding", + "target" + ], + "instructions": "INSTRUCTIONS-mage" +} \ No newline at end of file diff --git a/examples/go/go-multi-rpc/tools/tools.go b/examples/go/go-multi-rpc/tools/tools.go new file mode 100644 index 0000000..6d969b3 --- /dev/null +++ b/examples/go/go-multi-rpc/tools/tools.go @@ -0,0 +1,6 @@ +//go:build tools +// +build tools + +package tools + +import _ "github.com/magefile/mage/mg" From 4e91a11a4fabab3ec896543b7f81301ed13ee18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 27 Aug 2024 13:58:26 +0200 Subject: [PATCH 3/3] clippy --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 13d68e8..f94dd19 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,7 +41,7 @@ impl Examples for GolemExamples { { let example = parse_example( &lang, - &lang_dir.path(), + lang_dir.path(), Path::new("INSTRUCTIONS"), &adapters_path, example_dir.path(), @@ -282,7 +282,7 @@ fn parse_example( wit_deps, wit_deps_targets: metadata .wit_deps_paths - .map(|dirs| dirs.iter().map(|dir| PathBuf::from(dir)).collect()), + .map(|dirs| dirs.iter().map(PathBuf::from).collect()), exclude: metadata.exclude.iter().cloned().collect(), } }