From 2899a29ae475113418bfb6fef2cddeb444110dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 28 Oct 2021 09:08:47 -0400 Subject: [PATCH 01/37] feat: implement offchain http host functions --- lib/runtime/wasmer/imports.go | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 69496b3c32..92e9af1b0e 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -89,6 +89,7 @@ package wasmer // extern int64_t ext_offchain_submit_transaction_version_1(void *context, int64_t a); // extern int64_t ext_offchain_timestamp_version_1(void *context); // extern void ext_offchain_sleep_until_version_1(void *context, int64_t a); +// extern int64_t ext_offchain_http_request_start_version_1(void *context, int64_t a, int64_t b, int64_t c); // // extern void ext_storage_append_version_1(void *context, int64_t a, int64_t b); // extern int64_t ext_storage_changes_root_version_1(void *context, int64_t a); @@ -1682,6 +1683,40 @@ func ext_offchain_sleep_until_version_1(_ unsafe.Pointer, deadline C.int64_t) { logger.Warn("unimplemented") } +//export ext_offchain_http_request_start_version_1 +func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, meta C.int64_t) C.int64_t { + logger.Debug("[ext_offchain_submit_transaction_version_1] executing...") + instanceContext := wasm.IntoInstanceContext(context) + + encHTTPMethod := asMemorySlice(instanceContext, methodSpan) + encURI := asMemorySlice(instanceContext, uriSpan) + encMetaParams := asMemorySlice(instanceContext, meta) + + var ( + HTTPMethod string + URI string + MetaParams []byte + ) + + if err := scale.Unmarshal(encHTTPMethod, &HTTPMethod); err != nil { + logger.Error("[ext_offchain_http_request_start_version_1] failed to decode http method data", "error", err) + } + + if err := scale.Unmarshal(encURI, &URI); err != nil { + logger.Error("[ext_offchain_http_request_start_version_1] failed to decode http method data", "error", err) + } + + if err := scale.Unmarshal(encMetaParams, &MetaParams); err != nil { + logger.Error("[ext_offchain_http_request_start_version_1] failed to decode http method data", "error", err) + } + + fmt.Println(HTTPMethod) + fmt.Println(HTTPMethod) + fmt.Println(MetaParams) + + return 0 +} + func storageAppend(storage runtime.Storage, key, valueToAppend []byte) error { nextLength := big.NewInt(1) var valueRes []byte @@ -2305,6 +2340,10 @@ func ImportsNodeRuntime() (*wasm.Imports, error) { //nolint if err != nil { return nil, err } + _, err = imports.Append("ext_offchain_http_request_start_version_1", ext_offchain_http_request_start_version_1, C.ext_offchain_http_request_start_version_1) + if err != nil { + return nil, err + } _, err = imports.Append("ext_sandbox_instance_teardown_version_1", ext_sandbox_instance_teardown_version_1, C.ext_sandbox_instance_teardown_version_1) if err != nil { return nil, err From 8f7a58106d781f6167355d55f64b76e4b8e22889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 29 Oct 2021 15:08:56 -0400 Subject: [PATCH 02/37] chore: decoding Result --- lib/runtime/constants.go | 2 +- lib/runtime/offchain/httpset.go | 74 +++++++++++++++++++++++++ lib/runtime/test_helpers.go | 12 ++--- lib/runtime/wasmer/imports.go | 53 ++++++++++-------- lib/runtime/wasmer/imports_test.go | 87 +++++++++++++++++++++++++----- 5 files changed, 186 insertions(+), 42 deletions(-) create mode 100644 lib/runtime/offchain/httpset.go diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index db4c7d96dc..b0833363ec 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -46,7 +46,7 @@ const ( // v0.8 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" - HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME_FP = "/Users/eclesiojunior/w3f/eclesio-polkaspec/test/runtimes/hostapi/target/release/wbuild/hostapi-runtime/hostapi_runtime.compact.wasm" HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/80fa2be272820731b5159e9dc2a3eec3cca02b4d/test/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go new file mode 100644 index 0000000000..6aec07f4be --- /dev/null +++ b/lib/runtime/offchain/httpset.go @@ -0,0 +1,74 @@ +package offchain + +import ( + "errors" + "net/http" + "sync" +) + +var ( + once sync.Once + HTTPSet *Set + + _ = OnceHTTPSet() +) + +type Set struct { + mtx *sync.Mutex + list []*http.Request +} + +// OnceHTTPSet +func OnceHTTPSet() *Set { + once.Do(func() { + HTTPSet = &Set{ + mtx: new(sync.Mutex), + list: make([]*http.Request, 0), + } + }) + + return HTTPSet +} + +// StartRequest create a new request using the method and the uri, adds the request into the list +// and then return the position of the request inside the list +func (p *Set) StartRequest(method, uri string) (int16, error) { + p.mtx.Lock() + defer p.mtx.Unlock() + + req, err := http.NewRequest(method, uri, nil) + if err != nil { + return 0, err + } + + p.list = append(p.list, req) + return int16(len(p.list) - 1), nil +} + +func (p *Set) ExecRequest(id int) error { + if len(p.list) <= id { + return errors.New("http list does not contains id %v") + } + + req := p.list[id] + + client := new(http.Client) + _, err := client.Do(req) + return err +} + +// Remove just remove a expecific request from list +func (p *Set) Remove(id int) { + p.mtx.Lock() + defer p.mtx.Unlock() + + p.list = append(p.list[:id], p.list[id+1:]...) +} + +func (p *Set) Get(id int) *http.Request { + if len(p.list) <= id { + return nil + } + + return p.list[id] +} diff --git a/lib/runtime/test_helpers.go b/lib/runtime/test_helpers.go index c6e969e69d..766ede553c 100644 --- a/lib/runtime/test_helpers.go +++ b/lib/runtime/test_helpers.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "net/http" "os" - "path" "testing" "github.com/ChainSafe/chaindb" @@ -72,11 +71,12 @@ func GetRuntimeVars(targetRuntime string) (string, string) { // GetAbsolutePath returns the completePath for a given targetDir func GetAbsolutePath(targetDir string) string { - dir, err := os.Getwd() - if err != nil { - panic("failed to get current working directory") - } - return path.Join(dir, targetDir) + // dir, err := os.Getwd() + // if err != nil { + // panic("failed to get current working directory") + // } + // return path.Join(dir, targetDir) + return targetDir } // GetRuntimeBlob checks if the test wasm @testRuntimeFilePath exists and if not, it fetches it from @testRuntimeURL diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 92e9af1b0e..1986533678 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -48,7 +48,7 @@ package wasmer // // extern int32_t ext_trie_blake2_256_root_version_1(void *context, int64_t a); // extern int32_t ext_trie_blake2_256_ordered_root_version_1(void *context, int64_t a); -// extern int32_t ext_trie_blake2_256_verify_proof_version_1(void *context, int32_t a, int64_t b, int64_t c, int64_t d); +// extern int32_t ext_trie_blake2_256_verify_proof_version_1(void *context, int64_t a, int64_t b, int64_t c, int64_t d); // // extern int64_t ext_misc_runtime_version_version_1(void *context, int64_t a); // extern void ext_misc_print_hex_version_1(void *context, int64_t a); @@ -128,6 +128,7 @@ import ( "github.com/ChainSafe/gossamer/lib/crypto/secp256k1" "github.com/ChainSafe/gossamer/lib/crypto/sr25519" "github.com/ChainSafe/gossamer/lib/runtime" + "github.com/ChainSafe/gossamer/lib/runtime/offchain" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/transaction" "github.com/ChainSafe/gossamer/lib/trie" @@ -900,7 +901,7 @@ func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan } //export ext_trie_blake2_256_verify_proof_version_1 -func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a C.int32_t, b, c, d C.int64_t) C.int32_t { +func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a, b, c, d C.int64_t) C.int32_t { logger.Debug("[ext_trie_blake2_256_verify_proof_version_1] executing...") logger.Warn("[ext_trie_blake2_256_verify_proof_version_1] unimplemented") return 0 @@ -1684,37 +1685,45 @@ func ext_offchain_sleep_until_version_1(_ unsafe.Pointer, deadline C.int64_t) { } //export ext_offchain_http_request_start_version_1 -func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, meta C.int64_t) C.int64_t { +func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, metaSpan C.int64_t) C.int64_t { logger.Debug("[ext_offchain_submit_transaction_version_1] executing...") - instanceContext := wasm.IntoInstanceContext(context) - encHTTPMethod := asMemorySlice(instanceContext, methodSpan) - encURI := asMemorySlice(instanceContext, uriSpan) - encMetaParams := asMemorySlice(instanceContext, meta) + instanceContext := wasm.IntoInstanceContext(context) - var ( - HTTPMethod string - URI string - MetaParams []byte - ) + httpMethod := asMemorySlice(instanceContext, methodSpan) + uri := asMemorySlice(instanceContext, uriSpan) - if err := scale.Unmarshal(encHTTPMethod, &HTTPMethod); err != nil { - logger.Error("[ext_offchain_http_request_start_version_1] failed to decode http method data", "error", err) + reqID, err := offchain.HTTPSet.StartRequest(string(httpMethod), string(uri)) + if err != nil { + logger.Error("[ext_offchain_http_request_start_version_1] failed to start request", "error", err) + ptr, _ := toWasmMemoryResult(instanceContext, nil) + return C.int64_t(ptr) } - if err := scale.Unmarshal(encURI, &URI); err != nil { - logger.Error("[ext_offchain_http_request_start_version_1] failed to decode http method data", "error", err) + result := scale.NewResult(int16(0), string("")) + err = result.Set(scale.OK, reqID) + + if err != nil { + logger.Error("[ext_offchain_http_request_start_version_1] failed to set request start result", "error", err) + ptr, _ := toWasmMemoryResult(instanceContext, nil) + return C.int64_t(ptr) } - if err := scale.Unmarshal(encMetaParams, &MetaParams); err != nil { - logger.Error("[ext_offchain_http_request_start_version_1] failed to decode http method data", "error", err) + encResult, err := scale.Marshal(result) + if err != nil { + logger.Error("[ext_offchain_http_request_start_version_1] failed to encode request start result", "error", err) + ptr, _ := toWasmMemoryResult(instanceContext, nil) + return C.int64_t(ptr) } - fmt.Println(HTTPMethod) - fmt.Println(HTTPMethod) - fmt.Println(MetaParams) + ptr, err := toWasmMemory(instanceContext, encResult) + if err != nil { + logger.Error("[ext_offchain_http_request_start_version_1] failed to create ptr", "error", err) + ptr, _ := toWasmMemoryResult(instanceContext, nil) + return C.int64_t(ptr) + } - return 0 + return C.int64_t(ptr) } func storageAppend(storage runtime.Storage, key, valueToAppend []byte) error { diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index 865dfce136..7350fa50cb 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -19,7 +19,7 @@ package wasmer import ( "bytes" "encoding/binary" - "os" + "fmt" "sort" "testing" @@ -35,7 +35,6 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" - log "github.com/ChainSafe/log15" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wasmerio/go-ext-wasm/wasmer" @@ -45,19 +44,19 @@ var testChildKey = []byte("childKey") var testKey = []byte("key") var testValue = []byte("value") -func TestMain(m *testing.M) { - wasmFilePaths, err := runtime.GenerateRuntimeWasmFile() - if err != nil { - log.Error("failed to generate runtime wasm file", err) - os.Exit(1) - } +// func TestMain(m *testing.M) { +// wasmFilePaths, err := runtime.GenerateRuntimeWasmFile() +// if err != nil { +// log.Error("failed to generate runtime wasm file", err) +// os.Exit(1) +// } - // Start all tests - code := m.Run() +// // Start all tests +// code := m.Run() - runtime.RemoveFiles(wasmFilePaths) - os.Exit(code) -} +// runtime.RemoveFiles(wasmFilePaths) +// os.Exit(code) +// } func Test_ext_hashing_blake2_128_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) @@ -232,6 +231,68 @@ func Test_ext_offchain_local_storage_clear_version_1_Local(t *testing.T) { require.Nil(t, val) } +func Test_ext_offchain_http_request_start_version_1(t *testing.T) { + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + encMethod, err := scale.Marshal([]byte("GET")) + require.NoError(t, err) + + encUri, err := scale.Marshal([]byte("https://chainsafe.io")) + require.NoError(t, err) + + var optMeta *[]byte + encMeta, err := scale.Marshal(optMeta) + require.NoError(t, err) + + params := append([]byte{}, encMethod...) + params = append(params, encUri...) + params = append(params, encMeta...) + + inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) + require.NoError(t, err) + + ret, err := inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) + require.NoError(t, err) + + fmt.Println(ret) + + reqID := scale.NewResult(int16(0), string("")) + err = scale.Unmarshal(ret, &reqID) + require.NoError(t, err) + + fmt.Println(reqID) + require.NoError(t, err) + + ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) + require.NoError(t, err) + + fmt.Println(ret) + + reqID = scale.NewResult(int16(0), string("")) + err = scale.Unmarshal(ret, &reqID) + require.NoError(t, err) + + fmt.Println(reqID) + require.NoError(t, err) + //require.Equal(t, int16(1), ok.(int16)) + + // ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) + // require.NoError(t, err) + + // err = scale.Unmarshal(ret, &reqID) + // require.NoError(t, err) + + // fmt.Println(reqID, ret) + // require.Equal(t, int16(1), reqID) + + // ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) + // require.NoError(t, err) + + // err = scale.Unmarshal(ret, &reqID) + // require.NoError(t, err) + // require.Equal(t, int16(2), reqID) +} + func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) From 4d6167fe2a69334b4b4341884ecc0cda19342cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Sun, 31 Oct 2021 14:13:13 -0400 Subject: [PATCH 03/37] chore: adjust result encoding/decoding --- lib/runtime/offchain/httpset.go | 29 ++++++-------------- lib/runtime/wasmer/imports.go | 2 +- lib/runtime/wasmer/imports_test.go | 44 ++++++++++++------------------ 3 files changed, 26 insertions(+), 49 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 6aec07f4be..2e45cb46fb 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -1,7 +1,6 @@ package offchain import ( - "errors" "net/http" "sync" ) @@ -15,7 +14,7 @@ var ( type Set struct { mtx *sync.Mutex - list []*http.Request + reqs []*http.Request } // OnceHTTPSet @@ -23,7 +22,7 @@ func OnceHTTPSet() *Set { once.Do(func() { HTTPSet = &Set{ mtx: new(sync.Mutex), - list: make([]*http.Request, 0), + reqs: make([]*http.Request, 0), } }) @@ -41,34 +40,22 @@ func (p *Set) StartRequest(method, uri string) (int16, error) { return 0, err } - p.list = append(p.list, req) - return int16(len(p.list) - 1), nil + p.reqs = append(p.reqs, req) + return int16(len(p.reqs) - 1), nil } -func (p *Set) ExecRequest(id int) error { - if len(p.list) <= id { - return errors.New("http list does not contains id %v") - } - - req := p.list[id] - - client := new(http.Client) - _, err := client.Do(req) - return err -} - -// Remove just remove a expecific request from list +// Remove just remove a expecific request from reqs func (p *Set) Remove(id int) { p.mtx.Lock() defer p.mtx.Unlock() - p.list = append(p.list[:id], p.list[id+1:]...) + p.reqs = append(p.reqs[:id], p.reqs[id+1:]...) } func (p *Set) Get(id int) *http.Request { - if len(p.list) <= id { + if len(p.reqs) <= id { return nil } - return p.list[id] + return p.reqs[id] } diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 1986533678..500b3224b9 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1700,7 +1700,7 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa return C.int64_t(ptr) } - result := scale.NewResult(int16(0), string("")) + result := scale.NewResult(int16(0), nil) err = result.Set(scale.OK, reqID) if err != nil { diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index 7350fa50cb..484f2e25a1 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -19,7 +19,6 @@ package wasmer import ( "bytes" "encoding/binary" - "fmt" "sort" "testing" @@ -248,49 +247,40 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { params = append(params, encUri...) params = append(params, encMeta...) - inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) - require.NoError(t, err) + resReqID := scale.NewResult(int16(0), nil) + // start request number 0 ret, err := inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) require.NoError(t, err) - fmt.Println(ret) - - reqID := scale.NewResult(int16(0), string("")) - err = scale.Unmarshal(ret, &reqID) + err = scale.Unmarshal(ret, &resReqID) require.NoError(t, err) - fmt.Println(reqID) + requestNumber, err := resReqID.Unwrap() require.NoError(t, err) + require.Equal(t, int16(0), requestNumber) + // start request number 1 ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) require.NoError(t, err) - fmt.Println(ret) - - reqID = scale.NewResult(int16(0), string("")) - err = scale.Unmarshal(ret, &reqID) + err = scale.Unmarshal(ret, &resReqID) require.NoError(t, err) - fmt.Println(reqID) + requestNumber, err = resReqID.Unwrap() require.NoError(t, err) - //require.Equal(t, int16(1), ok.(int16)) - - // ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) - // require.NoError(t, err) - - // err = scale.Unmarshal(ret, &reqID) - // require.NoError(t, err) + require.Equal(t, int16(1), requestNumber) - // fmt.Println(reqID, ret) - // require.Equal(t, int16(1), reqID) + // start request number 2 + ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) + require.NoError(t, err) - // ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) - // require.NoError(t, err) + err = scale.Unmarshal(ret, &resReqID) + require.NoError(t, err) - // err = scale.Unmarshal(ret, &reqID) - // require.NoError(t, err) - // require.Equal(t, int16(2), reqID) + requestNumber, err = resReqID.Unwrap() + require.NoError(t, err) + require.Equal(t, int16(2), requestNumber) } func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { From faa5752ba4592f191633a107f01d4c4d978284f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Sun, 31 Oct 2021 14:19:31 -0400 Subject: [PATCH 04/37] chore: add export comment on Get --- lib/runtime/constants.go | 2 +- lib/runtime/offchain/httpset.go | 1 + lib/runtime/test_helpers.go | 12 ++++++------ lib/runtime/wasmer/imports_test.go | 24 +++++++++++++----------- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index b0833363ec..db4c7d96dc 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -46,7 +46,7 @@ const ( // v0.8 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" - HOST_API_TEST_RUNTIME_FP = "/Users/eclesiojunior/w3f/eclesio-polkaspec/test/runtimes/hostapi/target/release/wbuild/hostapi-runtime/hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/80fa2be272820731b5159e9dc2a3eec3cca02b4d/test/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 2e45cb46fb..1e60b97e7e 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -52,6 +52,7 @@ func (p *Set) Remove(id int) { p.reqs = append(p.reqs[:id], p.reqs[id+1:]...) } +// Get returns a request or nil if request not found func (p *Set) Get(id int) *http.Request { if len(p.reqs) <= id { return nil diff --git a/lib/runtime/test_helpers.go b/lib/runtime/test_helpers.go index 766ede553c..c6e969e69d 100644 --- a/lib/runtime/test_helpers.go +++ b/lib/runtime/test_helpers.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "net/http" "os" + "path" "testing" "github.com/ChainSafe/chaindb" @@ -71,12 +72,11 @@ func GetRuntimeVars(targetRuntime string) (string, string) { // GetAbsolutePath returns the completePath for a given targetDir func GetAbsolutePath(targetDir string) string { - // dir, err := os.Getwd() - // if err != nil { - // panic("failed to get current working directory") - // } - // return path.Join(dir, targetDir) - return targetDir + dir, err := os.Getwd() + if err != nil { + panic("failed to get current working directory") + } + return path.Join(dir, targetDir) } // GetRuntimeBlob checks if the test wasm @testRuntimeFilePath exists and if not, it fetches it from @testRuntimeURL diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index 484f2e25a1..280dcbb5d7 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -19,6 +19,7 @@ package wasmer import ( "bytes" "encoding/binary" + "os" "sort" "testing" @@ -34,6 +35,7 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + log "github.com/ChainSafe/log15" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wasmerio/go-ext-wasm/wasmer" @@ -43,19 +45,19 @@ var testChildKey = []byte("childKey") var testKey = []byte("key") var testValue = []byte("value") -// func TestMain(m *testing.M) { -// wasmFilePaths, err := runtime.GenerateRuntimeWasmFile() -// if err != nil { -// log.Error("failed to generate runtime wasm file", err) -// os.Exit(1) -// } +func TestMain(m *testing.M) { + wasmFilePaths, err := runtime.GenerateRuntimeWasmFile() + if err != nil { + log.Error("failed to generate runtime wasm file", err) + os.Exit(1) + } -// // Start all tests -// code := m.Run() + // Start all tests + code := m.Run() -// runtime.RemoveFiles(wasmFilePaths) -// os.Exit(code) -// } + runtime.RemoveFiles(wasmFilePaths) + os.Exit(code) +} func Test_ext_hashing_blake2_128_version_1(t *testing.T) { inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) From e08300e1b152498ea37d4b69cce828034fc821c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 2 Nov 2021 09:09:35 -0400 Subject: [PATCH 05/37] chore: change to map and update test wasm --- lib/runtime/constants.go | 2 +- lib/runtime/offchain/httpset.go | 84 +++++++++++++++++++++++---------- lib/runtime/types.go | 18 +++---- lib/runtime/wasmer/imports.go | 5 +- lib/runtime/wasmer/instance.go | 18 +++---- 5 files changed, 83 insertions(+), 44 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index db4c7d96dc..216a91e2b5 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -47,7 +47,7 @@ const ( // v0.8 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/80fa2be272820731b5159e9dc2a3eec3cca02b4d/test/hostapi_runtime.compact.wasm?raw=true" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/838a050dfe394a733e0ec95427289750fea8e381/test/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 1e60b97e7e..1ec2559303 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -1,32 +1,61 @@ package offchain import ( + "errors" "net/http" "sync" ) -var ( - once sync.Once - HTTPSet *Set +const maxConcurrentRequests = 1000 - _ = OnceHTTPSet() +var ( + errIntBufferEmpty = errors.New("int buffer exhausted") + errIntBufferFull = errors.New("int buffer is full") + errRequestIDNotAvailable = errors.New("request id not available") ) -type Set struct { - mtx *sync.Mutex - reqs []*http.Request +type intBuffer chan int16 + +func newIntBuffer(buffSize int16) *intBuffer { + b := make(chan int16, buffSize) + for i := int16(0); i < buffSize; i++ { + b <- i + } + + intb := intBuffer(b) + return &intb } -// OnceHTTPSet -func OnceHTTPSet() *Set { - once.Do(func() { - HTTPSet = &Set{ - mtx: new(sync.Mutex), - reqs: make([]*http.Request, 0), - } - }) +func (b *intBuffer) Get() (int16, error) { + select { + case v := <-*b: + return v, nil + default: + return 0, errIntBufferEmpty + } +} + +func (b *intBuffer) Put(i int16) error { + select { + case *b <- i: + return nil + default: + return errIntBufferFull + } +} - return HTTPSet +type Set struct { + mtx *sync.Mutex + reqs map[int16]*http.Request + idBuff *intBuffer +} + +func NewSet() *Set { + return &Set{ + mtx: new(sync.Mutex), + reqs: make(map[int16]*http.Request), + idBuff: newIntBuffer(maxConcurrentRequests), + } } // StartRequest create a new request using the method and the uri, adds the request into the list @@ -35,28 +64,33 @@ func (p *Set) StartRequest(method, uri string) (int16, error) { p.mtx.Lock() defer p.mtx.Unlock() + id, err := p.idBuff.Get() + if err != nil { + return 0, err + } + + if _, ok := p.reqs[id]; ok { + return 0, errRequestIDNotAvailable + } + req, err := http.NewRequest(method, uri, nil) if err != nil { return 0, err } - p.reqs = append(p.reqs, req) - return int16(len(p.reqs) - 1), nil + p.reqs[id] = req + return id, nil } // Remove just remove a expecific request from reqs -func (p *Set) Remove(id int) { +func (p *Set) Remove(id int16) { p.mtx.Lock() defer p.mtx.Unlock() - p.reqs = append(p.reqs[:id], p.reqs[id+1:]...) + delete(p.reqs, id) } // Get returns a request or nil if request not found -func (p *Set) Get(id int) *http.Request { - if len(p.reqs) <= id { - return nil - } - +func (p *Set) Get(id int16) *http.Request { return p.reqs[id] } diff --git a/lib/runtime/types.go b/lib/runtime/types.go index 6a131104af..10b9dd1eb8 100644 --- a/lib/runtime/types.go +++ b/lib/runtime/types.go @@ -19,6 +19,7 @@ package runtime import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/keystore" + "github.com/ChainSafe/gossamer/lib/runtime/offchain" log "github.com/ChainSafe/log15" ) @@ -72,14 +73,15 @@ type InstanceConfig struct { // Context is the context for the wasm interpreter's imported functions type Context struct { - Storage Storage - Allocator *FreeingBumpHeapAllocator - Keystore *keystore.GlobalKeystore - Validator bool - NodeStorage NodeStorage - Network BasicNetwork - Transaction TransactionState - SigVerifier *SignatureVerifier + Storage Storage + Allocator *FreeingBumpHeapAllocator + Keystore *keystore.GlobalKeystore + Validator bool + NodeStorage NodeStorage + Network BasicNetwork + Transaction TransactionState + SigVerifier *SignatureVerifier + OffchainHTTPSet *offchain.Set } // NewValidateTransactionError returns an error based on a return value from TaggedTransactionQueueValidateTransaction diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 500b3224b9..af9b2847b1 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -128,7 +128,6 @@ import ( "github.com/ChainSafe/gossamer/lib/crypto/secp256k1" "github.com/ChainSafe/gossamer/lib/crypto/sr25519" "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/runtime/offchain" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/transaction" "github.com/ChainSafe/gossamer/lib/trie" @@ -1693,7 +1692,9 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa httpMethod := asMemorySlice(instanceContext, methodSpan) uri := asMemorySlice(instanceContext, uriSpan) - reqID, err := offchain.HTTPSet.StartRequest(string(httpMethod), string(uri)) + runtimeCtx := instanceContext.Data().(*runtime.Context) + reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri)) + if err != nil { logger.Error("[ext_offchain_http_request_start_version_1] failed to start request", "error", err) ptr, _ := toWasmMemoryResult(instanceContext, nil) diff --git a/lib/runtime/wasmer/instance.go b/lib/runtime/wasmer/instance.go index f0c1fcfd3d..7f9f5df79c 100644 --- a/lib/runtime/wasmer/instance.go +++ b/lib/runtime/wasmer/instance.go @@ -25,6 +25,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/runtime" + "github.com/ChainSafe/gossamer/lib/runtime/offchain" "github.com/ChainSafe/gossamer/lib/trie" log "github.com/ChainSafe/log15" @@ -149,14 +150,15 @@ func newInstance(code []byte, cfg *Config) (*Instance, error) { allocator := runtime.NewAllocator(instance.Memory, heapBase) runtimeCtx := &runtime.Context{ - Storage: cfg.Storage, - Allocator: allocator, - Keystore: cfg.Keystore, - Validator: cfg.Role == byte(4), - NodeStorage: cfg.NodeStorage, - Network: cfg.Network, - Transaction: cfg.Transaction, - SigVerifier: runtime.NewSignatureVerifier(), + Storage: cfg.Storage, + Allocator: allocator, + Keystore: cfg.Keystore, + Validator: cfg.Role == byte(4), + NodeStorage: cfg.NodeStorage, + Network: cfg.Network, + Transaction: cfg.Transaction, + SigVerifier: runtime.NewSignatureVerifier(), + OffchainHTTPSet: offchain.NewSet(), } logger.Debug("NewInstance", "runtimeCtx", runtimeCtx) From 70e6351a1054787a6f9d6e2b6faec1ca40b181e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 2 Nov 2021 09:10:25 -0400 Subject: [PATCH 06/37] chore: use request id buffer --- lib/runtime/offchain/httpset.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 1ec2559303..bebb9fc159 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -14,19 +14,19 @@ var ( errRequestIDNotAvailable = errors.New("request id not available") ) -type intBuffer chan int16 +type requestIDBuffer chan int16 -func newIntBuffer(buffSize int16) *intBuffer { +func newIntBuffer(buffSize int16) *requestIDBuffer { b := make(chan int16, buffSize) for i := int16(0); i < buffSize; i++ { b <- i } - intb := intBuffer(b) + intb := requestIDBuffer(b) return &intb } -func (b *intBuffer) Get() (int16, error) { +func (b *requestIDBuffer) Get() (int16, error) { select { case v := <-*b: return v, nil @@ -35,7 +35,7 @@ func (b *intBuffer) Get() (int16, error) { } } -func (b *intBuffer) Put(i int16) error { +func (b *requestIDBuffer) Put(i int16) error { select { case *b <- i: return nil @@ -47,7 +47,7 @@ func (b *intBuffer) Put(i int16) error { type Set struct { mtx *sync.Mutex reqs map[int16]*http.Request - idBuff *intBuffer + idBuff *requestIDBuffer } func NewSet() *Set { From ad8c0bb0c67b94f639b41dd0dd26ba4b1536c7ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 2 Nov 2021 09:13:34 -0400 Subject: [PATCH 07/37] chore: change to NewHTTPSet --- lib/runtime/offchain/httpset.go | 2 +- lib/runtime/wasmer/instance.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index bebb9fc159..3360db8415 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -50,7 +50,7 @@ type Set struct { idBuff *requestIDBuffer } -func NewSet() *Set { +func NewHTTPSet() *Set { return &Set{ mtx: new(sync.Mutex), reqs: make(map[int16]*http.Request), diff --git a/lib/runtime/wasmer/instance.go b/lib/runtime/wasmer/instance.go index 7f9f5df79c..cafa936d76 100644 --- a/lib/runtime/wasmer/instance.go +++ b/lib/runtime/wasmer/instance.go @@ -158,7 +158,7 @@ func newInstance(code []byte, cfg *Config) (*Instance, error) { Network: cfg.Network, Transaction: cfg.Transaction, SigVerifier: runtime.NewSignatureVerifier(), - OffchainHTTPSet: offchain.NewSet(), + OffchainHTTPSet: offchain.NewHTTPSet(), } logger.Debug("NewInstance", "runtimeCtx", runtimeCtx) From 5606599b23ef34a304aad94dbf11a4fb353ca81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 2 Nov 2021 09:14:55 -0400 Subject: [PATCH 08/37] chore: add export comment --- lib/runtime/offchain/httpset.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 3360db8415..9dbdc62319 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -50,6 +50,8 @@ type Set struct { idBuff *requestIDBuffer } +// NewHTTPSet creates a offchain http set that can be used +// by runtime as HTTP clients, the max concurrent requests is 1000 func NewHTTPSet() *Set { return &Set{ mtx: new(sync.Mutex), From 81c0fb9cd80db802a436ded4c7ff105941d43717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 2 Nov 2021 12:21:24 -0400 Subject: [PATCH 09/37] chore: use pkg/scale to encode Result to wasm memory --- lib/runtime/wasmer/imports.go | 45 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index af9b2847b1..31cdd10cd5 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1685,42 +1685,30 @@ func ext_offchain_sleep_until_version_1(_ unsafe.Pointer, deadline C.int64_t) { //export ext_offchain_http_request_start_version_1 func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, metaSpan C.int64_t) C.int64_t { - logger.Debug("[ext_offchain_submit_transaction_version_1] executing...") + logger.Debug("executing...") instanceContext := wasm.IntoInstanceContext(context) httpMethod := asMemorySlice(instanceContext, methodSpan) uri := asMemorySlice(instanceContext, uriSpan) + result := scale.NewResult(int16(0), nil) + runtimeCtx := instanceContext.Data().(*runtime.Context) reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri)) if err != nil { - logger.Error("[ext_offchain_http_request_start_version_1] failed to start request", "error", err) - ptr, _ := toWasmMemoryResult(instanceContext, nil) + logger.Error("failed to start request", "error", err) + ptr, _ := fromResultToWasm(instanceContext, &result, scale.Err, nil) return C.int64_t(ptr) } - result := scale.NewResult(int16(0), nil) - err = result.Set(scale.OK, reqID) - + ptr, err := fromResultToWasm(instanceContext, &result, scale.OK, reqID) if err != nil { - logger.Error("[ext_offchain_http_request_start_version_1] failed to set request start result", "error", err) - ptr, _ := toWasmMemoryResult(instanceContext, nil) - return C.int64_t(ptr) - } + logger.Error("failed to start request", "error", err) - encResult, err := scale.Marshal(result) - if err != nil { - logger.Error("[ext_offchain_http_request_start_version_1] failed to encode request start result", "error", err) - ptr, _ := toWasmMemoryResult(instanceContext, nil) - return C.int64_t(ptr) - } - - ptr, err := toWasmMemory(instanceContext, encResult) - if err != nil { - logger.Error("[ext_offchain_http_request_start_version_1] failed to create ptr", "error", err) - ptr, _ := toWasmMemoryResult(instanceContext, nil) + errRes := scale.NewResult(int16(0), nil) + ptr, _ := fromResultToWasm(instanceContext, &errRes, scale.Err, nil) return C.int64_t(ptr) } @@ -2080,6 +2068,21 @@ func toWasmMemoryOptional(context wasm.InstanceContext, data []byte) (int64, err return toWasmMemory(context, enc) } +// fromResultToWasm wraps a slice in Result type using the pkg/scale.Result and copies result to wasm memory +func fromResultToWasm(context wasm.InstanceContext, res *scale.Result, m scale.ResultMode, data interface{}) (int64, error) { + err := res.Set(m, data) + if err != nil { + return 0, err + } + + encResult, err := scale.Marshal(res) + if err != nil { + return 0, err + } + + return toWasmMemory(context, encResult) +} + // Wraps slice in Result type and copies result to wasm memory. Returns resulting 64bit span descriptor func toWasmMemoryResult(context wasm.InstanceContext, data []byte) (int64, error) { var res *rtype.Result From fb21f114be97080610f7c7d5ba13365b66c01af1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 2 Nov 2021 12:27:18 -0400 Subject: [PATCH 10/37] chore: update naming and fix lint warns --- lib/runtime/offchain/httpset.go | 22 ++++++++++++---------- lib/runtime/types.go | 2 +- lib/runtime/wasmer/imports.go | 4 ++-- lib/runtime/wasmer/imports_test.go | 4 ++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 9dbdc62319..c275445b44 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -10,7 +10,7 @@ const maxConcurrentRequests = 1000 var ( errIntBufferEmpty = errors.New("int buffer exhausted") - errIntBufferFull = errors.New("int buffer is full") + errIntBufferFull = errors.New("int buffer is full") //nolint:unused errRequestIDNotAvailable = errors.New("request id not available") ) @@ -26,7 +26,7 @@ func newIntBuffer(buffSize int16) *requestIDBuffer { return &intb } -func (b *requestIDBuffer) Get() (int16, error) { +func (b *requestIDBuffer) get() (int16, error) { select { case v := <-*b: return v, nil @@ -35,7 +35,8 @@ func (b *requestIDBuffer) Get() (int16, error) { } } -func (b *requestIDBuffer) Put(i int16) error { +// nolint:unused +func (b *requestIDBuffer) put(i int16) error { select { case *b <- i: return nil @@ -44,7 +45,8 @@ func (b *requestIDBuffer) Put(i int16) error { } } -type Set struct { +// HTTPSet holds a pool of concurrent http request calls +type HTTPSet struct { mtx *sync.Mutex reqs map[int16]*http.Request idBuff *requestIDBuffer @@ -52,8 +54,8 @@ type Set struct { // NewHTTPSet creates a offchain http set that can be used // by runtime as HTTP clients, the max concurrent requests is 1000 -func NewHTTPSet() *Set { - return &Set{ +func NewHTTPSet() *HTTPSet { + return &HTTPSet{ mtx: new(sync.Mutex), reqs: make(map[int16]*http.Request), idBuff: newIntBuffer(maxConcurrentRequests), @@ -62,11 +64,11 @@ func NewHTTPSet() *Set { // StartRequest create a new request using the method and the uri, adds the request into the list // and then return the position of the request inside the list -func (p *Set) StartRequest(method, uri string) (int16, error) { +func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { p.mtx.Lock() defer p.mtx.Unlock() - id, err := p.idBuff.Get() + id, err := p.idBuff.get() if err != nil { return 0, err } @@ -85,7 +87,7 @@ func (p *Set) StartRequest(method, uri string) (int16, error) { } // Remove just remove a expecific request from reqs -func (p *Set) Remove(id int16) { +func (p *HTTPSet) Remove(id int16) { p.mtx.Lock() defer p.mtx.Unlock() @@ -93,6 +95,6 @@ func (p *Set) Remove(id int16) { } // Get returns a request or nil if request not found -func (p *Set) Get(id int16) *http.Request { +func (p *HTTPSet) Get(id int16) *http.Request { return p.reqs[id] } diff --git a/lib/runtime/types.go b/lib/runtime/types.go index 10b9dd1eb8..8a4a734146 100644 --- a/lib/runtime/types.go +++ b/lib/runtime/types.go @@ -81,7 +81,7 @@ type Context struct { Network BasicNetwork Transaction TransactionState SigVerifier *SignatureVerifier - OffchainHTTPSet *offchain.Set + OffchainHTTPSet *offchain.HTTPSet } // NewValidateTransactionError returns an error based on a return value from TaggedTransactionQueueValidateTransaction diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 64c21d6e35..0c4fe12a0f 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1701,8 +1701,8 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa logger.Error("failed to start request", "error", err) errRes := scale.NewResult(int16(0), nil) - ptr, _ := fromResultToWasm(instanceContext, &errRes, scale.Err, nil) - return C.int64_t(ptr) + errPtr, _ := fromResultToWasm(instanceContext, &errRes, scale.Err, nil) + return C.int64_t(errPtr) } return C.int64_t(ptr) diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index f03b18a401..e9b71d3def 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -238,7 +238,7 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { encMethod, err := scale.Marshal([]byte("GET")) require.NoError(t, err) - encUri, err := scale.Marshal([]byte("https://chainsafe.io")) + encURI, err := scale.Marshal([]byte("https://chainsafe.io")) require.NoError(t, err) var optMeta *[]byte @@ -246,7 +246,7 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { require.NoError(t, err) params := append([]byte{}, encMethod...) - params = append(params, encUri...) + params = append(params, encURI...) params = append(params, encMeta...) resReqID := scale.NewResult(int16(0), nil) From ee372c48078672e392db4445186bb6cc41b52b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 3 Nov 2021 17:42:25 -0400 Subject: [PATCH 11/37] chore: use buffer.put when remove http request --- lib/runtime/offchain/httpset.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index c275445b44..e57a83fdd4 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -10,10 +10,11 @@ const maxConcurrentRequests = 1000 var ( errIntBufferEmpty = errors.New("int buffer exhausted") - errIntBufferFull = errors.New("int buffer is full") //nolint:unused + errIntBufferFull = errors.New("int buffer is full") errRequestIDNotAvailable = errors.New("request id not available") ) +// requestIDBuffer created to controll the amount of available ids and to avoid use of randon id generation type requestIDBuffer chan int16 func newIntBuffer(buffSize int16) *requestIDBuffer { @@ -35,7 +36,6 @@ func (b *requestIDBuffer) get() (int16, error) { } } -// nolint:unused func (b *requestIDBuffer) put(i int16) error { select { case *b <- i: @@ -87,11 +87,13 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { } // Remove just remove a expecific request from reqs -func (p *HTTPSet) Remove(id int16) { +func (p *HTTPSet) Remove(id int16) error { p.mtx.Lock() defer p.mtx.Unlock() delete(p.reqs, id) + + return p.idBuff.put(id) } // Get returns a request or nil if request not found From 78bfd0bdc6319116d02dcc010beeab97881ef15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 3 Nov 2021 17:45:36 -0400 Subject: [PATCH 12/37] chore: add more comments --- lib/runtime/offchain/httpset.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index e57a83fdd4..bd4e9802e8 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -14,9 +14,10 @@ var ( errRequestIDNotAvailable = errors.New("request id not available") ) -// requestIDBuffer created to controll the amount of available ids and to avoid use of randon id generation +// requestIDBuffer created to controll the amount of available non-duplicated ids type requestIDBuffer chan int16 +// newIntBuffer creates the request id buffer starting from 0 till @buffSize (by default @buffSize is 1000) func newIntBuffer(buffSize int16) *requestIDBuffer { b := make(chan int16, buffSize) for i := int16(0); i < buffSize; i++ { From 42ff50eba950949390f0f4e34a72bd61140b3a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 3 Nov 2021 17:56:18 -0400 Subject: [PATCH 13/37] chore: add unit tests --- lib/runtime/offchain/httpset_test.go | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 lib/runtime/offchain/httpset_test.go diff --git a/lib/runtime/offchain/httpset_test.go b/lib/runtime/offchain/httpset_test.go new file mode 100644 index 0000000000..c51253f517 --- /dev/null +++ b/lib/runtime/offchain/httpset_test.go @@ -0,0 +1,47 @@ +package offchain + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +const defaultTestURI = "http://example.url" + +func TestHTTPSetLimit(t *testing.T) { + t.Parallel() + + set := NewHTTPSet() + var err error + for i := 0; i < maxConcurrentRequests+1; i++ { + _, err = set.StartRequest(http.MethodGet, defaultTestURI) + } + + require.ErrorIs(t, errIntBufferEmpty, err) +} + +func TestHTTPSet_StartRequest_NotAvailableID(t *testing.T) { + t.Parallel() + + set := NewHTTPSet() + set.reqs[0] = &http.Request{} + + _, err := set.StartRequest(http.MethodGet, defaultTestURI) + require.ErrorIs(t, errRequestIDNotAvailable, err) +} + +func TestHTTPSetGet(t *testing.T) { + t.Parallel() + + set := NewHTTPSet() + + id, err := set.StartRequest(http.MethodGet, defaultTestURI) + require.NoError(t, err) + + req := set.Get(id) + require.NotNil(t, req) + + require.Equal(t, http.MethodGet, req.Method) + require.Equal(t, defaultTestURI, req.URL.String()) +} From 13d540a4adfe7a393c02bcdd2c05926043cb31e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 3 Nov 2021 17:56:58 -0400 Subject: [PATCH 14/37] chore: fix misspelling --- lib/runtime/offchain/httpset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index bd4e9802e8..7591c1da8e 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -14,7 +14,7 @@ var ( errRequestIDNotAvailable = errors.New("request id not available") ) -// requestIDBuffer created to controll the amount of available non-duplicated ids +// requestIDBuffer created to control the amount of available non-duplicated ids type requestIDBuffer chan int16 // newIntBuffer creates the request id buffer starting from 0 till @buffSize (by default @buffSize is 1000) From 60a1f116a6bd0d406e0a2184949b8f6ac1a0eab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 3 Nov 2021 19:35:11 -0400 Subject: [PATCH 15/37] chore: fix scale marshal to encode Result instead of Option --- lib/runtime/offchain/httpset.go | 4 ++-- lib/runtime/wasmer/imports.go | 32 ++++++++---------------------- lib/runtime/wasmer/imports_test.go | 6 +++--- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 7591c1da8e..942561384f 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -17,10 +17,10 @@ var ( // requestIDBuffer created to control the amount of available non-duplicated ids type requestIDBuffer chan int16 -// newIntBuffer creates the request id buffer starting from 0 till @buffSize (by default @buffSize is 1000) +// newIntBuffer creates the request id buffer starting from 1 till @buffSize (by default @buffSize is 1000) func newIntBuffer(buffSize int16) *requestIDBuffer { b := make(chan int16, buffSize) - for i := int16(0); i < buffSize; i++ { + for i := int16(1); i <= buffSize; i++ { b <- i } diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 0c4fe12a0f..b8499e8859 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1691,19 +1691,18 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri)) if err != nil { + result.Set(scale.Err, nil) + + enc, _ := scale.Marshal(result) + ptr, _ := toWasmMemory(instanceContext, enc) + logger.Error("failed to start request", "error", err) - ptr, _ := fromResultToWasm(instanceContext, &result, scale.Err, nil) return C.int64_t(ptr) } - ptr, err := fromResultToWasm(instanceContext, &result, scale.OK, reqID) - if err != nil { - logger.Error("failed to start request", "error", err) - - errRes := scale.NewResult(int16(0), nil) - errPtr, _ := fromResultToWasm(instanceContext, &errRes, scale.Err, nil) - return C.int64_t(errPtr) - } + result.Set(scale.OK, reqID) + enc, _ := scale.Marshal(result) + ptr, _ := toWasmMemory(instanceContext, enc) return C.int64_t(ptr) } @@ -2080,21 +2079,6 @@ func toWasmMemoryOptional(context wasm.InstanceContext, data []byte) (int64, err return toWasmMemory(context, enc) } -// fromResultToWasm wraps a slice in Result type using the pkg/scale.Result and copies result to wasm memory -func fromResultToWasm(context wasm.InstanceContext, res *scale.Result, m scale.ResultMode, data interface{}) (int64, error) { - err := res.Set(m, data) - if err != nil { - return 0, err - } - - encResult, err := scale.Marshal(res) - if err != nil { - return 0, err - } - - return toWasmMemory(context, encResult) -} - // Wraps slice in Result type and copies result to wasm memory. Returns resulting 64bit span descriptor func toWasmMemoryResult(context wasm.InstanceContext, data []byte) (int64, error) { var res *rtype.Result diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index e9b71d3def..7869cb11ef 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -260,7 +260,7 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { requestNumber, err := resReqID.Unwrap() require.NoError(t, err) - require.Equal(t, int16(0), requestNumber) + require.Equal(t, int16(1), requestNumber) // start request number 1 ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) @@ -271,7 +271,7 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { requestNumber, err = resReqID.Unwrap() require.NoError(t, err) - require.Equal(t, int16(1), requestNumber) + require.Equal(t, int16(2), requestNumber) // start request number 2 ret, err = inst.Exec("rtm_ext_offchain_http_request_start_version_1", params) @@ -282,7 +282,7 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { requestNumber, err = resReqID.Unwrap() require.NoError(t, err) - require.Equal(t, int16(2), requestNumber) + require.Equal(t, int16(3), requestNumber) } func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { From 4cd379f16d984ecfdc54d8913f2f107f7e3bdf45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 4 Nov 2021 11:32:29 -0400 Subject: [PATCH 16/37] chore: ignore uneeded error --- lib/runtime/wasmer/imports.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index b8499e8859..f25d47cad8 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1691,16 +1691,12 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri)) if err != nil { - result.Set(scale.Err, nil) - - enc, _ := scale.Marshal(result) - ptr, _ := toWasmMemory(instanceContext, enc) - logger.Error("failed to start request", "error", err) - return C.int64_t(ptr) + _ = result.Set(scale.Err, nil) + } else { + _ = result.Set(scale.OK, reqID) } - result.Set(scale.OK, reqID) enc, _ := scale.Marshal(result) ptr, _ := toWasmMemory(instanceContext, enc) From f1130ad8c74593b62146375a0354f060c9c92a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 4 Nov 2021 14:40:59 -0400 Subject: [PATCH 17/37] chore: fix unused params --- lib/runtime/wasmer/imports.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index f25d47cad8..b652d92537 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -895,7 +895,7 @@ func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan } //export ext_trie_blake2_256_verify_proof_version_1 -func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a, b, c, d C.int64_t) C.int32_t { +func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, _, _, _, _ C.int64_t) C.int32_t { logger.Debug("[ext_trie_blake2_256_verify_proof_version_1] executing...") logger.Warn("[ext_trie_blake2_256_verify_proof_version_1] unimplemented") return 0 @@ -1677,7 +1677,7 @@ func ext_offchain_sleep_until_version_1(_ unsafe.Pointer, deadline C.int64_t) { } //export ext_offchain_http_request_start_version_1 -func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, metaSpan C.int64_t) C.int64_t { +func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, _ C.int64_t) C.int64_t { logger.Debug("executing...") instanceContext := wasm.IntoInstanceContext(context) From 5f980abb3a803e7571309d8b30b9d557c4ac7d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 4 Nov 2021 15:35:21 -0400 Subject: [PATCH 18/37] chore: cannot remove unused params --- lib/runtime/wasmer/imports.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index b652d92537..95e439929c 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -895,7 +895,7 @@ func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan } //export ext_trie_blake2_256_verify_proof_version_1 -func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, _, _, _, _ C.int64_t) C.int32_t { +func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a, b, c, d C.int64_t) C.int32_t { //nolint logger.Debug("[ext_trie_blake2_256_verify_proof_version_1] executing...") logger.Warn("[ext_trie_blake2_256_verify_proof_version_1] unimplemented") return 0 @@ -1677,7 +1677,7 @@ func ext_offchain_sleep_until_version_1(_ unsafe.Pointer, deadline C.int64_t) { } //export ext_offchain_http_request_start_version_1 -func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, _ C.int64_t) C.int64_t { +func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, metaSpan C.int64_t) C.int64_t { //nolint logger.Debug("executing...") instanceContext := wasm.IntoInstanceContext(context) From c8913d29f969544a95b4fa786fc8e235e7b2d2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 4 Nov 2021 16:09:18 -0400 Subject: [PATCH 19/37] chore: ignore deepsource errors --- lib/runtime/wasmer/imports.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 95e439929c..636160d082 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -895,7 +895,7 @@ func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan } //export ext_trie_blake2_256_verify_proof_version_1 -func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a, b, c, d C.int64_t) C.int32_t { //nolint +func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a, b, c, d C.int64_t) C.int32_t { // skipcq: RVV-B0012 logger.Debug("[ext_trie_blake2_256_verify_proof_version_1] executing...") logger.Warn("[ext_trie_blake2_256_verify_proof_version_1] unimplemented") return 0 @@ -1677,7 +1677,7 @@ func ext_offchain_sleep_until_version_1(_ unsafe.Pointer, deadline C.int64_t) { } //export ext_offchain_http_request_start_version_1 -func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, metaSpan C.int64_t) C.int64_t { //nolint +func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpan, uriSpan, metaSpan C.int64_t) C.int64_t { // skipcq: RVV-B0012 logger.Debug("executing...") instanceContext := wasm.IntoInstanceContext(context) From fcb5da93e25a8c62a2d99e3540e7d3ee092af9e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 4 Nov 2021 16:35:59 -0400 Subject: [PATCH 20/37] chore: add parallel to wasmer tests --- lib/runtime/wasmer/imports_test.go | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index 7869cb11ef..e921dd503c 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -60,6 +60,7 @@ func TestMain(m *testing.M) { } func Test_ext_hashing_blake2_128_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) data := []byte("helloworld") @@ -79,6 +80,7 @@ func Test_ext_hashing_blake2_128_version_1(t *testing.T) { } func Test_ext_hashing_blake2_256_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) data := []byte("helloworld") @@ -98,6 +100,7 @@ func Test_ext_hashing_blake2_256_version_1(t *testing.T) { } func Test_ext_hashing_keccak_256_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) data := []byte("helloworld") @@ -117,6 +120,7 @@ func Test_ext_hashing_keccak_256_version_1(t *testing.T) { } func Test_ext_hashing_twox_128_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) data := []byte("helloworld") @@ -136,6 +140,7 @@ func Test_ext_hashing_twox_128_version_1(t *testing.T) { } func Test_ext_hashing_twox_64_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) data := []byte("helloworld") @@ -155,6 +160,7 @@ func Test_ext_hashing_twox_64_version_1(t *testing.T) { } func Test_ext_hashing_sha2_256_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) data := []byte("helloworld") @@ -173,6 +179,7 @@ func Test_ext_hashing_sha2_256_version_1(t *testing.T) { } func Test_ext_storage_clear_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -189,6 +196,7 @@ func Test_ext_storage_clear_version_1(t *testing.T) { } func Test_ext_offchain_local_storage_clear_version_1_Persistent(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("key1") @@ -211,6 +219,7 @@ func Test_ext_offchain_local_storage_clear_version_1_Persistent(t *testing.T) { } func Test_ext_offchain_local_storage_clear_version_1_Local(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("key1") @@ -233,6 +242,7 @@ func Test_ext_offchain_local_storage_clear_version_1_Local(t *testing.T) { } func Test_ext_offchain_http_request_start_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) encMethod, err := scale.Marshal([]byte("GET")) @@ -286,6 +296,7 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { } func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("static") @@ -308,6 +319,7 @@ func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { } func Test_ext_storage_clear_prefix_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -330,6 +342,7 @@ func Test_ext_storage_clear_prefix_version_1(t *testing.T) { } func Test_ext_storage_clear_prefix_version_2(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -402,6 +415,7 @@ func Test_ext_storage_clear_prefix_version_2(t *testing.T) { } func Test_ext_storage_get_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -422,6 +436,7 @@ func Test_ext_storage_get_version_1(t *testing.T) { } func Test_ext_storage_exists_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -445,6 +460,7 @@ func Test_ext_storage_exists_version_1(t *testing.T) { } func Test_ext_storage_next_key_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -467,6 +483,7 @@ func Test_ext_storage_next_key_version_1(t *testing.T) { } func Test_ext_storage_read_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -495,6 +512,7 @@ func Test_ext_storage_read_version_1(t *testing.T) { } func Test_ext_storage_read_version_1_again(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -524,6 +542,7 @@ func Test_ext_storage_read_version_1_again(t *testing.T) { } func Test_ext_storage_read_version_1_OffsetLargerThanValue(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -552,6 +571,7 @@ func Test_ext_storage_read_version_1_OffsetLargerThanValue(t *testing.T) { } func Test_ext_storage_root_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) ret, err := inst.Exec("rtm_ext_storage_root_version_1", []byte{}) @@ -566,6 +586,7 @@ func Test_ext_storage_root_version_1(t *testing.T) { } func Test_ext_storage_set_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -584,6 +605,7 @@ func Test_ext_storage_set_version_1(t *testing.T) { } func Test_ext_offline_index_set_version_1(t *testing.T) { + t.Parallel() // TODO this currently fails with error could not find exported function, add rtm_ func to tester wasm (#1026) t.Skip() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) @@ -605,6 +627,7 @@ func Test_ext_offline_index_set_version_1(t *testing.T) { } func Test_ext_crypto_ed25519_generate_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) idData := []byte(keystore.AccoName) @@ -648,6 +671,7 @@ func Test_ext_crypto_ed25519_generate_version_1(t *testing.T) { } func Test_ext_crypto_ed25519_public_keys_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) idData := []byte(keystore.DumyName) @@ -682,6 +706,7 @@ func Test_ext_crypto_ed25519_public_keys_version_1(t *testing.T) { } func Test_ext_crypto_ed25519_sign_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) kp, err := ed25519.GenerateKeypair() @@ -720,6 +745,7 @@ func Test_ext_crypto_ed25519_sign_version_1(t *testing.T) { } func Test_ext_crypto_ed25519_verify_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) kp, err := ed25519.GenerateKeypair() @@ -787,6 +813,7 @@ func Test_ext_crypto_ecdsa_verify_version_2(t *testing.T) { } func Test_ext_crypto_ecdsa_verify_version_2_Table(t *testing.T) { + t.Parallel() testCases := map[string]struct { sig []byte msg []byte @@ -844,6 +871,7 @@ func Test_ext_crypto_ecdsa_verify_version_2_Table(t *testing.T) { } func Test_ext_crypto_sr25519_generate_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) idData := []byte(keystore.AccoName) @@ -876,6 +904,7 @@ func Test_ext_crypto_sr25519_generate_version_1(t *testing.T) { } func Test_ext_crypto_secp256k1_ecdsa_recover_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) msgData := []byte("Hello world!") @@ -919,6 +948,7 @@ func Test_ext_crypto_secp256k1_ecdsa_recover_version_1(t *testing.T) { } func Test_ext_crypto_secp256k1_ecdsa_recover_compressed_version_1(t *testing.T) { + t.Parallel() t.Skip("host API tester does not yet contain rtm_ext_crypto_secp256k1_ecdsa_recover_compressed_version_1") inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) @@ -962,6 +992,7 @@ func Test_ext_crypto_secp256k1_ecdsa_recover_compressed_version_1(t *testing.T) } func Test_ext_crypto_sr25519_public_keys_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) idData := []byte(keystore.DumyName) @@ -996,6 +1027,7 @@ func Test_ext_crypto_sr25519_public_keys_version_1(t *testing.T) { } func Test_ext_crypto_sr25519_sign_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) kp, err := sr25519.GenerateKeypair() @@ -1036,6 +1068,7 @@ func Test_ext_crypto_sr25519_sign_version_1(t *testing.T) { } func Test_ext_crypto_sr25519_verify_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) kp, err := sr25519.GenerateKeypair() @@ -1068,6 +1101,7 @@ func Test_ext_crypto_sr25519_verify_version_1(t *testing.T) { } func Test_ext_default_child_storage_read_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) @@ -1104,6 +1138,7 @@ func Test_ext_default_child_storage_read_version_1(t *testing.T) { } func Test_ext_default_child_storage_clear_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) @@ -1132,6 +1167,7 @@ func Test_ext_default_child_storage_clear_version_1(t *testing.T) { } func Test_ext_default_child_storage_clear_prefix_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) prefix := []byte("key") @@ -1173,6 +1209,7 @@ func Test_ext_default_child_storage_clear_prefix_version_1(t *testing.T) { } func Test_ext_default_child_storage_exists_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) @@ -1197,6 +1234,7 @@ func Test_ext_default_child_storage_exists_version_1(t *testing.T) { } func Test_ext_default_child_storage_get_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) @@ -1221,6 +1259,7 @@ func Test_ext_default_child_storage_get_version_1(t *testing.T) { } func Test_ext_default_child_storage_next_key_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testKeyValuePair := []struct { @@ -1258,6 +1297,7 @@ func Test_ext_default_child_storage_next_key_version_1(t *testing.T) { } func Test_ext_default_child_storage_root_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) @@ -1290,6 +1330,7 @@ func Test_ext_default_child_storage_root_version_1(t *testing.T) { } func Test_ext_default_child_storage_set_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) @@ -1318,6 +1359,7 @@ func Test_ext_default_child_storage_set_version_1(t *testing.T) { } func Test_ext_default_child_storage_storage_kill_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) err := inst.ctx.Storage.SetChild(testChildKey, trie.NewEmptyTrie()) @@ -1339,6 +1381,7 @@ func Test_ext_default_child_storage_storage_kill_version_1(t *testing.T) { } func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() @@ -1372,6 +1415,7 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_all(t *testing. } func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() @@ -1405,6 +1449,7 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_1(t *testing.T) } func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() @@ -1436,6 +1481,7 @@ func Test_ext_default_child_storage_storage_kill_version_2_limit_none(t *testing } func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) tr := trie.NewEmptyTrie() @@ -1482,6 +1528,7 @@ func Test_ext_default_child_storage_storage_kill_version_3(t *testing.T) { } func Test_ext_storage_append_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testkey := []byte("noot") @@ -1530,6 +1577,7 @@ func Test_ext_storage_append_version_1(t *testing.T) { } func Test_ext_storage_append_version_1_again(t *testing.T) { + t.Parallel() DefaultTestLogLvl = 5 inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) @@ -1579,6 +1627,7 @@ func Test_ext_storage_append_version_1_again(t *testing.T) { } func Test_ext_trie_blake2_256_ordered_root_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testvalues := []string{"static", "even-keeled", "Future-proofed"} @@ -1597,6 +1646,7 @@ func Test_ext_trie_blake2_256_ordered_root_version_1(t *testing.T) { } func Test_ext_trie_blake2_256_root_version_1(t *testing.T) { + t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) testinput := []string{"noot", "was", "here", "??"} From 9d2b7c5c9845702aff695a4913fcd48ac3d78c04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 5 Nov 2021 15:44:26 -0400 Subject: [PATCH 21/37] chore: implementing offchain http request add header --- lib/runtime/constants.go | 2 +- lib/runtime/offchain/httpset.go | 58 ++++++++++++---- lib/runtime/offchain/httpset_test.go | 54 ++++++++++++++- lib/runtime/test_helpers.go | 12 ++-- lib/runtime/wasmer/imports.go | 31 +++++++++ lib/runtime/wasmer/imports_test.go | 98 ++++++++++++++++++++++++---- 6 files changed, 220 insertions(+), 35 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index af2820eac4..ebb27884df 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -46,7 +46,7 @@ const ( // v0.9 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" - HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" + HOST_API_TEST_RUNTIME_FP = "/Users/eclesiojunior/w3f/eclesio-polkaspec/test/runtimes/hostapi/target/release/wbuild/hostapi-runtime/hostapi_runtime.compact.wasm" HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/838a050dfe394a733e0ec95427289750fea8e381/test/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 942561384f..7125a85dd1 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -2,6 +2,7 @@ package offchain import ( "errors" + "fmt" "net/http" "sync" ) @@ -12,45 +13,70 @@ var ( errIntBufferEmpty = errors.New("int buffer exhausted") errIntBufferFull = errors.New("int buffer is full") errRequestIDNotAvailable = errors.New("request id not available") + errInvalidRequest = errors.New("request is invalid") + errRequestAlreadyStarted = errors.New("request has already started") + errInvalidHeaderKey = errors.New("invalid header key") ) // requestIDBuffer created to control the amount of available non-duplicated ids type requestIDBuffer chan int16 // newIntBuffer creates the request id buffer starting from 1 till @buffSize (by default @buffSize is 1000) -func newIntBuffer(buffSize int16) *requestIDBuffer { +func newIntBuffer(buffSize int16) requestIDBuffer { b := make(chan int16, buffSize) for i := int16(1); i <= buffSize; i++ { b <- i } - intb := requestIDBuffer(b) - return &intb + return b } -func (b *requestIDBuffer) get() (int16, error) { +func (b requestIDBuffer) get() (int16, error) { select { - case v := <-*b: + case v := <-b: return v, nil default: return 0, errIntBufferEmpty } } -func (b *requestIDBuffer) put(i int16) error { +func (b requestIDBuffer) put(i int16) error { select { - case *b <- i: + case b <- i: return nil default: return errIntBufferFull } } +type OffchainRequest struct { + Request *http.Request + invalid, waiting bool +} + +// AddHeader add a new header into @req property only if request is valid or has not started yet +func (r *OffchainRequest) AddHeader(k, v string) error { + if r.invalid { + return errInvalidRequest + } + + if r.waiting { + return errRequestAlreadyStarted + } + + if k == "" { + return fmt.Errorf("%w: %s", errInvalidHeaderKey, "empty header key") + } + + r.Request.Header.Add(k, v) + return nil +} + // HTTPSet holds a pool of concurrent http request calls type HTTPSet struct { mtx *sync.Mutex - reqs map[int16]*http.Request - idBuff *requestIDBuffer + reqs map[int16]*OffchainRequest + idBuff requestIDBuffer } // NewHTTPSet creates a offchain http set that can be used @@ -58,7 +84,7 @@ type HTTPSet struct { func NewHTTPSet() *HTTPSet { return &HTTPSet{ mtx: new(sync.Mutex), - reqs: make(map[int16]*http.Request), + reqs: make(map[int16]*OffchainRequest), idBuff: newIntBuffer(maxConcurrentRequests), } } @@ -83,7 +109,12 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { return 0, err } - p.reqs[id] = req + p.reqs[id] = &OffchainRequest{ + Request: req, + invalid: false, + waiting: false, + } + return id, nil } @@ -98,6 +129,9 @@ func (p *HTTPSet) Remove(id int16) error { } // Get returns a request or nil if request not found -func (p *HTTPSet) Get(id int16) *http.Request { +func (p *HTTPSet) Get(id int16) *OffchainRequest { + p.mtx.Lock() + defer p.mtx.Unlock() + return p.reqs[id] } diff --git a/lib/runtime/offchain/httpset_test.go b/lib/runtime/offchain/httpset_test.go index c51253f517..acfca8b79b 100644 --- a/lib/runtime/offchain/httpset_test.go +++ b/lib/runtime/offchain/httpset_test.go @@ -1,6 +1,7 @@ package offchain import ( + "fmt" "net/http" "testing" @@ -25,7 +26,7 @@ func TestHTTPSet_StartRequest_NotAvailableID(t *testing.T) { t.Parallel() set := NewHTTPSet() - set.reqs[0] = &http.Request{} + set.reqs[1] = &OffchainRequest{} _, err := set.StartRequest(http.MethodGet, defaultTestURI) require.ErrorIs(t, errRequestIDNotAvailable, err) @@ -42,6 +43,53 @@ func TestHTTPSetGet(t *testing.T) { req := set.Get(id) require.NotNil(t, req) - require.Equal(t, http.MethodGet, req.Method) - require.Equal(t, defaultTestURI, req.URL.String()) + require.Equal(t, http.MethodGet, req.req.Method) + require.Equal(t, defaultTestURI, req.req.URL.String()) +} + +func TestOffchainRequest_AddHeader(t *testing.T) { + t.Parallel() + + cases := map[string]struct { + offReq OffchainRequest + err error + headerK, headerV string + }{ + "should return invalid request": { + offReq: OffchainRequest{invalid: true}, + err: errInvalidRequest, + }, + "should return request already started": { + offReq: OffchainRequest{waiting: true}, + err: errRequestAlreadyStarted, + }, + "should add header": { + offReq: OffchainRequest{req: &http.Request{Header: make(http.Header)}}, + headerK: "key", + headerV: "value", + }, + "should return invalid empty header": { + offReq: OffchainRequest{req: &http.Request{Header: make(http.Header)}}, + headerK: "", + headerV: "value", + err: fmt.Errorf("%w: %s", errInvalidHeaderKey, "empty header key"), + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + err := tc.offReq.AddHeader(tc.headerK, tc.headerV) + + if tc.err != nil { + require.Error(t, err) + require.Equal(t, tc.err.Error(), err.Error()) + return + } + + got := tc.offReq.req.Header.Get(tc.headerK) + require.Equal(t, tc.headerV, got) + }) + } } diff --git a/lib/runtime/test_helpers.go b/lib/runtime/test_helpers.go index 59da6fd68d..4cff3d426d 100644 --- a/lib/runtime/test_helpers.go +++ b/lib/runtime/test_helpers.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "net/http" "os" - "path" "testing" "github.com/ChainSafe/chaindb" @@ -72,11 +71,12 @@ func GetRuntimeVars(targetRuntime string) (string, string) { // GetAbsolutePath returns the completePath for a given targetDir func GetAbsolutePath(targetDir string) string { - dir, err := os.Getwd() - if err != nil { - panic("failed to get current working directory") - } - return path.Join(dir, targetDir) + // dir, err := os.Getwd() + // if err != nil { + // panic("failed to get current working directory") + // } + // return path.Join(dir, targetDir) + return targetDir } // GetRuntimeBlob checks if the test wasm @testRuntimeFilePath exists and if not, it fetches it from @testRuntimeURL diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 636160d082..eabf203fa2 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -90,6 +90,7 @@ package wasmer // extern int64_t ext_offchain_timestamp_version_1(void *context); // extern void ext_offchain_sleep_until_version_1(void *context, int64_t a); // extern int64_t ext_offchain_http_request_start_version_1(void *context, int64_t a, int64_t b, int64_t c); +// extern int64_t ext_offchain_http_request_add_header_version_1(void *context, int32_t a, int64_t k, int64_t v); // // extern void ext_storage_append_version_1(void *context, int64_t a, int64_t b); // extern int64_t ext_storage_changes_root_version_1(void *context, int64_t a); @@ -1703,6 +1704,32 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa return C.int64_t(ptr) } +//export ext_offchain_http_request_add_header_version_1 +func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, reqID C.int32_t, keySpan, valueSpan C.int64_t) C.int64_t { + logger.Debug("executing...") + instanceContext := wasm.IntoInstanceContext(context) + + key := asMemorySlice(instanceContext, keySpan) + value := asMemorySlice(instanceContext, valueSpan) + + runtimeCtx := instanceContext.Data().(*runtime.Context) + offchainReq := runtimeCtx.OffchainHTTPSet.Get(int16(reqID)) + + result := scale.NewResult(nil, nil) + err := offchainReq.AddHeader(string(key), string(value)) + if err != nil { + logger.Error("failed to add request header", "error", err) + result.Set(scale.Err, nil) + } else { + result.Set(scale.OK, nil) + } + + enc, _ := scale.Marshal(result) + ptr, _ := toWasmMemory(instanceContext, enc) + + return C.int64_t(ptr) +} + func storageAppend(storage runtime.Storage, key, valueToAppend []byte) error { nextLength := big.NewInt(1) var valueRes []byte @@ -2366,6 +2393,10 @@ func ImportsNodeRuntime() (*wasm.Imports, error) { //nolint if err != nil { return nil, err } + _, err = imports.Append("ext_offchain_http_request_add_header_version_1", ext_offchain_http_request_add_header_version_1, C.ext_offchain_http_request_add_header_version_1) + if err != nil { + return nil, err + } _, err = imports.Append("ext_sandbox_instance_teardown_version_1", ext_sandbox_instance_teardown_version_1, C.ext_sandbox_instance_teardown_version_1) if err != nil { return nil, err diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index e921dd503c..e23d560417 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -19,11 +19,11 @@ package wasmer import ( "bytes" "encoding/binary" - "os" + "fmt" + "net/http" "sort" "testing" - log "github.com/ChainSafe/log15" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wasmerio/go-ext-wasm/wasmer" @@ -45,19 +45,19 @@ var testChildKey = []byte("childKey") var testKey = []byte("key") var testValue = []byte("value") -func TestMain(m *testing.M) { - wasmFilePaths, err := runtime.GenerateRuntimeWasmFile() - if err != nil { - log.Error("failed to generate runtime wasm file", err) - os.Exit(1) - } +// func TestMain(m *testing.M) { +// wasmFilePaths, err := runtime.GenerateRuntimeWasmFile() +// if err != nil { +// log.Error("failed to generate runtime wasm file", err) +// os.Exit(1) +// } - // Start all tests - code := m.Run() +// // Start all tests +// code := m.Run() - runtime.RemoveFiles(wasmFilePaths) - os.Exit(code) -} +// runtime.RemoveFiles(wasmFilePaths) +// os.Exit(code) +// } func Test_ext_hashing_blake2_128_version_1(t *testing.T) { t.Parallel() @@ -259,6 +259,8 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { params = append(params, encURI...) params = append(params, encMeta...) + fmt.Println(params) + resReqID := scale.NewResult(int16(0), nil) // start request number 0 @@ -295,6 +297,76 @@ func Test_ext_offchain_http_request_start_version_1(t *testing.T) { require.Equal(t, int16(3), requestNumber) } +func Test_ext_offchain_http_request_add_header(t *testing.T) { + t.Parallel() + + inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) + + cases := map[string]struct { + key, value string + expectedErr bool + }{ + "should add headers without problems": { + key: "SOME_HEADER_KEY", + value: "SOME_HEADER_VALUE", + expectedErr: false, + }, + + "should return a result error": { + key: "", + value: "", + expectedErr: true, + }, + } + + for tname, tcase := range cases { + t.Run(tname, func(t *testing.T) { + t.Parallel() + + reqID, err := inst.ctx.OffchainHTTPSet.StartRequest(http.MethodGet, "http://uri.example") + require.NoError(t, err) + + encID, err := scale.Marshal(uint32(reqID)) + require.NoError(t, err) + + encHeaderKey, err := scale.Marshal(tcase.key) + require.NoError(t, err) + + encHeaderValue, err := scale.Marshal(tcase.value) + require.NoError(t, err) + + params := append([]byte{}, encID...) + params = append(params, encHeaderKey...) + params = append(params, encHeaderValue...) + + ret, err := inst.Exec("rtm_ext_offchain_http_request_add_header_version_1", params) + require.NoError(t, err) + + gotResult := scale.NewResult(nil, nil) + err = scale.Unmarshal(ret, &gotResult) + require.NoError(t, err) + + ok, err := gotResult.Unwrap() + if tcase.expectedErr { + require.Error(t, err) + + offchainReq := inst.ctx.OffchainHTTPSet.Get(reqID) + gotValue := offchainReq.Request.Header.Get(tcase.key) + require.Empty(t, gotValue) + + } else { + require.NoError(t, err) + + offchainReq := inst.ctx.OffchainHTTPSet.Get(reqID) + gotValue := offchainReq.Request.Header.Get(tcase.key) + require.Equal(t, tcase.value, gotValue) + } + + require.Nil(t, ok) + }) + } +} + func Test_ext_storage_clear_prefix_version_1_hostAPI(t *testing.T) { t.Parallel() inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME) From dd8947278636397c844a1891d267c501651ee835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 5 Nov 2021 15:46:23 -0400 Subject: [PATCH 22/37] chore: remove dereferencing --- lib/runtime/offchain/httpset.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 942561384f..ca39dcde29 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -18,28 +18,27 @@ var ( type requestIDBuffer chan int16 // newIntBuffer creates the request id buffer starting from 1 till @buffSize (by default @buffSize is 1000) -func newIntBuffer(buffSize int16) *requestIDBuffer { +func newIntBuffer(buffSize int16) requestIDBuffer { b := make(chan int16, buffSize) for i := int16(1); i <= buffSize; i++ { b <- i } - intb := requestIDBuffer(b) - return &intb + return b } -func (b *requestIDBuffer) get() (int16, error) { +func (b requestIDBuffer) get() (int16, error) { select { - case v := <-*b: + case v := <-b: return v, nil default: return 0, errIntBufferEmpty } } -func (b *requestIDBuffer) put(i int16) error { +func (b requestIDBuffer) put(i int16) error { select { - case *b <- i: + case b <- i: return nil default: return errIntBufferFull @@ -50,7 +49,7 @@ func (b *requestIDBuffer) put(i int16) error { type HTTPSet struct { mtx *sync.Mutex reqs map[int16]*http.Request - idBuff *requestIDBuffer + idBuff requestIDBuffer } // NewHTTPSet creates a offchain http set that can be used @@ -99,5 +98,8 @@ func (p *HTTPSet) Remove(id int16) error { // Get returns a request or nil if request not found func (p *HTTPSet) Get(id int16) *http.Request { + p.mtx.Lock() + defer p.mtx.Unlock() + return p.reqs[id] } From 8624e09bd07f5228cc57239de91241e6bdc4e503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 8 Nov 2021 14:51:32 -0400 Subject: [PATCH 23/37] chore: fix param compatibility --- lib/runtime/constants.go | 2 +- lib/runtime/wasmer/imports.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index af2820eac4..b4945237fd 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -47,7 +47,7 @@ const ( // v0.9 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/838a050dfe394a733e0ec95427289750fea8e381/test/hostapi_runtime.compact.wasm?raw=true" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/b94d8c58ad6ea8bf827b0cae1645a999719c2bc7/test/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 636160d082..f75a1ee4c7 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -48,7 +48,7 @@ package wasmer // // extern int32_t ext_trie_blake2_256_root_version_1(void *context, int64_t a); // extern int32_t ext_trie_blake2_256_ordered_root_version_1(void *context, int64_t a); -// extern int32_t ext_trie_blake2_256_verify_proof_version_1(void *context, int64_t a, int64_t b, int64_t c, int64_t d); +// extern int32_t ext_trie_blake2_256_verify_proof_version_1(void *context, int32_t a, int64_t b, int64_t c, int64_t d); // // extern int64_t ext_misc_runtime_version_version_1(void *context, int64_t a); // extern void ext_misc_print_hex_version_1(void *context, int64_t a); @@ -895,7 +895,7 @@ func ext_trie_blake2_256_ordered_root_version_1(context unsafe.Pointer, dataSpan } //export ext_trie_blake2_256_verify_proof_version_1 -func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a, b, c, d C.int64_t) C.int32_t { // skipcq: RVV-B0012 +func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, a C.int32_t, b, c, d C.int64_t) C.int32_t { // skipcq: RVV-B0012 logger.Debug("[ext_trie_blake2_256_verify_proof_version_1] executing...") logger.Warn("[ext_trie_blake2_256_verify_proof_version_1] unimplemented") return 0 From 427c183a1ac3fa5aff57fb82950c6ac99d5bb953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 8 Nov 2021 14:58:20 -0400 Subject: [PATCH 24/37] chore: embed mutex iunto httpset struct --- lib/runtime/offchain/httpset.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index ca39dcde29..7379a540ef 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -47,7 +47,7 @@ func (b requestIDBuffer) put(i int16) error { // HTTPSet holds a pool of concurrent http request calls type HTTPSet struct { - mtx *sync.Mutex + *sync.Mutex reqs map[int16]*http.Request idBuff requestIDBuffer } @@ -56,17 +56,17 @@ type HTTPSet struct { // by runtime as HTTP clients, the max concurrent requests is 1000 func NewHTTPSet() *HTTPSet { return &HTTPSet{ - mtx: new(sync.Mutex), - reqs: make(map[int16]*http.Request), - idBuff: newIntBuffer(maxConcurrentRequests), + new(sync.Mutex), + make(map[int16]*http.Request), + newIntBuffer(maxConcurrentRequests), } } // StartRequest create a new request using the method and the uri, adds the request into the list // and then return the position of the request inside the list func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { - p.mtx.Lock() - defer p.mtx.Unlock() + p.Lock() + defer p.Unlock() id, err := p.idBuff.get() if err != nil { @@ -88,8 +88,8 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { // Remove just remove a expecific request from reqs func (p *HTTPSet) Remove(id int16) error { - p.mtx.Lock() - defer p.mtx.Unlock() + p.Lock() + defer p.Unlock() delete(p.reqs, id) @@ -98,8 +98,8 @@ func (p *HTTPSet) Remove(id int16) error { // Get returns a request or nil if request not found func (p *HTTPSet) Get(id int16) *http.Request { - p.mtx.Lock() - defer p.mtx.Unlock() + p.Lock() + defer p.Unlock() return p.reqs[id] } From b791f355e2b9fdf61c34b1018c2caba0756366ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 8 Nov 2021 15:00:26 -0400 Subject: [PATCH 25/37] chore: fix request field name --- lib/runtime/offchain/httpset_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/runtime/offchain/httpset_test.go b/lib/runtime/offchain/httpset_test.go index acfca8b79b..450c71d5f8 100644 --- a/lib/runtime/offchain/httpset_test.go +++ b/lib/runtime/offchain/httpset_test.go @@ -43,8 +43,8 @@ func TestHTTPSetGet(t *testing.T) { req := set.Get(id) require.NotNil(t, req) - require.Equal(t, http.MethodGet, req.req.Method) - require.Equal(t, defaultTestURI, req.req.URL.String()) + require.Equal(t, http.MethodGet, req.Request.Method) + require.Equal(t, defaultTestURI, req.Request.URL.String()) } func TestOffchainRequest_AddHeader(t *testing.T) { @@ -64,12 +64,12 @@ func TestOffchainRequest_AddHeader(t *testing.T) { err: errRequestAlreadyStarted, }, "should add header": { - offReq: OffchainRequest{req: &http.Request{Header: make(http.Header)}}, + offReq: OffchainRequest{Request: &http.Request{Header: make(http.Header)}}, headerK: "key", headerV: "value", }, "should return invalid empty header": { - offReq: OffchainRequest{req: &http.Request{Header: make(http.Header)}}, + offReq: OffchainRequest{Request: &http.Request{Header: make(http.Header)}}, headerK: "", headerV: "value", err: fmt.Errorf("%w: %s", errInvalidHeaderKey, "empty header key"), @@ -88,7 +88,7 @@ func TestOffchainRequest_AddHeader(t *testing.T) { return } - got := tc.offReq.req.Header.Get(tc.headerK) + got := tc.offReq.Request.Header.Get(tc.headerK) require.Equal(t, tc.headerV, got) }) } From 750caa99b3272d49c187d7586e907f63579874c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 9 Nov 2021 16:01:44 -0400 Subject: [PATCH 26/37] chore: update the hoost polkadot test runtime location --- lib/runtime/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index b4945237fd..c244d1c978 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -47,7 +47,7 @@ const ( // v0.9 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/b94d8c58ad6ea8bf827b0cae1645a999719c2bc7/test/hostapi_runtime.compact.wasm?raw=true" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/b94d8c58ad6ea8bf827b0cae1645a999719c2bc7/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" From c856a8759bf88e9e2f96e32617911fdff38244c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 9 Nov 2021 16:34:38 -0400 Subject: [PATCH 27/37] chore: use an updated host runtime test --- lib/runtime/constants.go | 2 +- lib/runtime/offchain/httpset.go | 6 +++--- lib/runtime/test_helpers.go | 12 ++++++------ lib/runtime/wasmer/imports_test.go | 24 +++++++++++++----------- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index c244d1c978..6c4d55de4a 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -47,7 +47,7 @@ const ( // v0.9 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/b94d8c58ad6ea8bf827b0cae1645a999719c2bc7/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/e8c35823226943552a891189e6937525c6b2453a/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 7c8e1aa36c..06e6f7c0ef 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -75,7 +75,7 @@ func (r *OffchainRequest) AddHeader(k, v string) error { // HTTPSet holds a pool of concurrent http request calls type HTTPSet struct { *sync.Mutex - reqs map[int16]*http.Request + reqs map[int16]*OffchainRequest idBuff requestIDBuffer } @@ -84,7 +84,7 @@ type HTTPSet struct { func NewHTTPSet() *HTTPSet { return &HTTPSet{ new(sync.Mutex), - make(map[int16]*http.Request), + make(map[int16]*OffchainRequest), newIntBuffer(maxConcurrentRequests), } } @@ -129,7 +129,7 @@ func (p *HTTPSet) Remove(id int16) error { } // Get returns a request or nil if request not found -func (p *HTTPSet) Get(id int16) *http.Request { +func (p *HTTPSet) Get(id int16) *OffchainRequest { p.Lock() defer p.Unlock() diff --git a/lib/runtime/test_helpers.go b/lib/runtime/test_helpers.go index 4cff3d426d..59da6fd68d 100644 --- a/lib/runtime/test_helpers.go +++ b/lib/runtime/test_helpers.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "net/http" "os" + "path" "testing" "github.com/ChainSafe/chaindb" @@ -71,12 +72,11 @@ func GetRuntimeVars(targetRuntime string) (string, string) { // GetAbsolutePath returns the completePath for a given targetDir func GetAbsolutePath(targetDir string) string { - // dir, err := os.Getwd() - // if err != nil { - // panic("failed to get current working directory") - // } - // return path.Join(dir, targetDir) - return targetDir + dir, err := os.Getwd() + if err != nil { + panic("failed to get current working directory") + } + return path.Join(dir, targetDir) } // GetRuntimeBlob checks if the test wasm @testRuntimeFilePath exists and if not, it fetches it from @testRuntimeURL diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index e23d560417..61eb6b1594 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -21,9 +21,11 @@ import ( "encoding/binary" "fmt" "net/http" + "os" "sort" "testing" + log "github.com/ChainSafe/log15" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wasmerio/go-ext-wasm/wasmer" @@ -45,19 +47,19 @@ var testChildKey = []byte("childKey") var testKey = []byte("key") var testValue = []byte("value") -// func TestMain(m *testing.M) { -// wasmFilePaths, err := runtime.GenerateRuntimeWasmFile() -// if err != nil { -// log.Error("failed to generate runtime wasm file", err) -// os.Exit(1) -// } +func TestMain(m *testing.M) { + wasmFilePaths, err := runtime.GenerateRuntimeWasmFile() + if err != nil { + log.Error("failed to generate runtime wasm file", err) + os.Exit(1) + } -// // Start all tests -// code := m.Run() + // Start all tests + code := m.Run() -// runtime.RemoveFiles(wasmFilePaths) -// os.Exit(code) -// } + runtime.RemoveFiles(wasmFilePaths) + os.Exit(code) +} func Test_ext_hashing_blake2_128_version_1(t *testing.T) { t.Parallel() From 85c470308f6ba808f5214e9f2282415e53921ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 9 Nov 2021 16:57:42 -0400 Subject: [PATCH 28/37] chore: fix lint warns --- lib/runtime/offchain/httpset.go | 2 ++ lib/runtime/wasmer/imports.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 06e6f7c0ef..3be73fe86b 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -49,6 +49,8 @@ func (b requestIDBuffer) put(i int16) error { } } +// OffchainRequest holds the request object and update the invalid and waiting status whenever +// the request starts or is waiting to be read type OffchainRequest struct { Request *http.Request invalid, waiting bool diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index a29978f4fd..2a0c3ed20f 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1719,9 +1719,9 @@ func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, reqI err := offchainReq.AddHeader(string(key), string(value)) if err != nil { logger.Error("failed to add request header", "error", err) - result.Set(scale.Err, nil) + _ = result.Set(scale.Err, nil) } else { - result.Set(scale.OK, nil) + _ = result.Set(scale.OK, nil) } enc, _ := scale.Marshal(result) From d203860c5c6ad6602170c3893e486faed9a0ca62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 9 Nov 2021 17:06:51 -0400 Subject: [PATCH 29/37] chore: rename OffchainRequest to Request --- lib/runtime/offchain/httpset.go | 14 +++++++------- lib/runtime/offchain/httpset_test.go | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 3be73fe86b..297b282d9e 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -49,15 +49,15 @@ func (b requestIDBuffer) put(i int16) error { } } -// OffchainRequest holds the request object and update the invalid and waiting status whenever +// Request holds the request object and update the invalid and waiting status whenever // the request starts or is waiting to be read -type OffchainRequest struct { +type Request struct { Request *http.Request invalid, waiting bool } // AddHeader add a new header into @req property only if request is valid or has not started yet -func (r *OffchainRequest) AddHeader(k, v string) error { +func (r *Request) AddHeader(k, v string) error { if r.invalid { return errInvalidRequest } @@ -77,7 +77,7 @@ func (r *OffchainRequest) AddHeader(k, v string) error { // HTTPSet holds a pool of concurrent http request calls type HTTPSet struct { *sync.Mutex - reqs map[int16]*OffchainRequest + reqs map[int16]*Request idBuff requestIDBuffer } @@ -86,7 +86,7 @@ type HTTPSet struct { func NewHTTPSet() *HTTPSet { return &HTTPSet{ new(sync.Mutex), - make(map[int16]*OffchainRequest), + make(map[int16]*Request), newIntBuffer(maxConcurrentRequests), } } @@ -111,7 +111,7 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { return 0, err } - p.reqs[id] = &OffchainRequest{ + p.reqs[id] = &Request{ Request: req, invalid: false, waiting: false, @@ -131,7 +131,7 @@ func (p *HTTPSet) Remove(id int16) error { } // Get returns a request or nil if request not found -func (p *HTTPSet) Get(id int16) *OffchainRequest { +func (p *HTTPSet) Get(id int16) *Request { p.Lock() defer p.Unlock() diff --git a/lib/runtime/offchain/httpset_test.go b/lib/runtime/offchain/httpset_test.go index 450c71d5f8..4a4d3b1687 100644 --- a/lib/runtime/offchain/httpset_test.go +++ b/lib/runtime/offchain/httpset_test.go @@ -26,7 +26,7 @@ func TestHTTPSet_StartRequest_NotAvailableID(t *testing.T) { t.Parallel() set := NewHTTPSet() - set.reqs[1] = &OffchainRequest{} + set.reqs[1] = &Request{} _, err := set.StartRequest(http.MethodGet, defaultTestURI) require.ErrorIs(t, errRequestIDNotAvailable, err) @@ -51,25 +51,25 @@ func TestOffchainRequest_AddHeader(t *testing.T) { t.Parallel() cases := map[string]struct { - offReq OffchainRequest + offReq Request err error headerK, headerV string }{ "should return invalid request": { - offReq: OffchainRequest{invalid: true}, + offReq: Request{invalid: true}, err: errInvalidRequest, }, "should return request already started": { - offReq: OffchainRequest{waiting: true}, + offReq: Request{waiting: true}, err: errRequestAlreadyStarted, }, "should add header": { - offReq: OffchainRequest{Request: &http.Request{Header: make(http.Header)}}, + offReq: Request{Request: &http.Request{Header: make(http.Header)}}, headerK: "key", headerV: "value", }, "should return invalid empty header": { - offReq: OffchainRequest{Request: &http.Request{Header: make(http.Header)}}, + offReq: Request{Request: &http.Request{Header: make(http.Header)}}, headerK: "", headerV: "value", err: fmt.Errorf("%w: %s", errInvalidHeaderKey, "empty header key"), From 281ed2217b9c655dd99a689047f85c280b12fcf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Thu, 11 Nov 2021 09:14:11 -0400 Subject: [PATCH 30/37] chore: update host commit hash --- lib/runtime/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index 6c4d55de4a..f298734ef1 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -47,7 +47,7 @@ const ( // v0.9 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/e8c35823226943552a891189e6937525c6b2453a/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/7edb16fa9df0b93e59489bf85d958a564d2787ec/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" From 18a2cc850bf42d2ad36648c2487e7c734bb6fcf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 11 Nov 2021 09:27:20 -0400 Subject: [PATCH 31/37] chore: update log --- lib/runtime/wasmer/imports.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index d585a892b4..35fe03206d 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1757,7 +1757,7 @@ func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, reqI result := scale.NewResult(nil, nil) err := offchainReq.AddHeader(string(key), string(value)) if err != nil { - logger.Error("failed to add request header", "error", err) + logger.Errorf("failed to add request header: %s", err) _ = result.Set(scale.Err, nil) } else { _ = result.Set(scale.OK, nil) From d02a4343e555db38e0bc180f5e45dafd3b868e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 16 Nov 2021 17:04:12 -0400 Subject: [PATCH 32/37] chore: address comments --- lib/runtime/offchain/httpset.go | 18 +++++----- lib/runtime/offchain/httpset_test.go | 6 +++- lib/runtime/wasmer/imports.go | 53 ++++++++++++++++++++++------ lib/runtime/wasmer/imports_test.go | 13 +++---- 4 files changed, 60 insertions(+), 30 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 16367132fe..e687c1f4e9 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "net/http" + "strings" "sync" ) @@ -16,7 +17,7 @@ var ( errIntBufferEmpty = errors.New("int buffer exhausted") errIntBufferFull = errors.New("int buffer is full") errRequestIDNotAvailable = errors.New("request id not available") - errInvalidRequest = errors.New("request is invalid") + errRequestInvalid = errors.New("request is invalid") errRequestAlreadyStarted = errors.New("request has already started") errInvalidHeaderKey = errors.New("invalid header key") ) @@ -59,21 +60,22 @@ type Request struct { invalid, waiting bool } -// AddHeader add a new header into @req property only if request is valid or has not started yet -func (r *Request) AddHeader(k, v string) error { +// AddHeader adds a new HTTP header into request property, only if request is valid and has not started yet +func (r *Request) AddHeader(name, value string) error { if r.invalid { - return errInvalidRequest + return errRequestInvalid } if r.waiting { return errRequestAlreadyStarted } - if k == "" { - return fmt.Errorf("%w: %s", errInvalidHeaderKey, "empty header key") + name = strings.TrimSpace(name) + if len(name) == 0 { + return fmt.Errorf("%w: empty header key", errInvalidHeaderKey) } - r.Request.Header.Add(k, v) + r.Request.Header.Add(name, value) return nil } @@ -116,8 +118,6 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { p.reqs[id] = &Request{ Request: req, - invalid: false, - waiting: false, } return id, nil diff --git a/lib/runtime/offchain/httpset_test.go b/lib/runtime/offchain/httpset_test.go index 063650d765..1698666a46 100644 --- a/lib/runtime/offchain/httpset_test.go +++ b/lib/runtime/offchain/httpset_test.go @@ -60,7 +60,7 @@ func TestOffchainRequest_AddHeader(t *testing.T) { }{ "should return invalid request": { offReq: Request{invalid: true}, - err: errInvalidRequest, + err: errRequestInvalid, }, "should return request already started": { offReq: Request{waiting: true}, @@ -80,6 +80,8 @@ func TestOffchainRequest_AddHeader(t *testing.T) { } for name, tc := range cases { + tc := tc + t.Run(name, func(t *testing.T) { t.Parallel() @@ -91,6 +93,8 @@ func TestOffchainRequest_AddHeader(t *testing.T) { return } + require.NoError(t, err) + got := tc.offReq.Request.Header.Get(tc.headerK) require.Equal(t, tc.headerV, got) }) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index ad3114f3d7..a9e5f15026 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1728,45 +1728,76 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa uri := asMemorySlice(instanceContext, uriSpan) result := scale.NewResult(int16(0), nil) + var resultSetErr error runtimeCtx := instanceContext.Data().(*runtime.Context) reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri)) if err != nil { logger.Errorf("failed to start request: %s", err) - _ = result.Set(scale.Err, nil) + resultSetErr = result.Set(scale.Err, nil) } else { - _ = result.Set(scale.OK, reqID) + resultSetErr = result.Set(scale.OK, reqID) } - enc, _ := scale.Marshal(result) - ptr, _ := toWasmMemory(instanceContext, enc) + if resultSetErr != nil { + logger.Errorf("failed to set the result data: %s", err) + return C.int64_t(0) + } + + enc, err := scale.Marshal(result) + if err != nil { + logger.Errorf("failed to scale marshal the result: %s", err) + return C.int64_t(0) + } + + ptr, err := toWasmMemory(instanceContext, enc) + if err != nil { + logger.Errorf("failed to allocate result on memory: %s", err) + return C.int64_t(0) + } return C.int64_t(ptr) } //export ext_offchain_http_request_add_header_version_1 -func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, reqID C.int32_t, keySpan, valueSpan C.int64_t) C.int64_t { +func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, reqID C.int32_t, nameSpan, valueSpan C.int64_t) C.int64_t { logger.Debug("executing...") instanceContext := wasm.IntoInstanceContext(context) - key := asMemorySlice(instanceContext, keySpan) + name := asMemorySlice(instanceContext, nameSpan) value := asMemorySlice(instanceContext, valueSpan) runtimeCtx := instanceContext.Data().(*runtime.Context) offchainReq := runtimeCtx.OffchainHTTPSet.Get(int16(reqID)) result := scale.NewResult(nil, nil) - err := offchainReq.AddHeader(string(key), string(value)) + var resultSetErr error + + err := offchainReq.AddHeader(string(name), string(value)) if err != nil { logger.Errorf("failed to add request header: %s", err) - _ = result.Set(scale.Err, nil) + resultSetErr = result.Set(scale.Err, nil) } else { - _ = result.Set(scale.OK, nil) + resultSetErr = result.Set(scale.OK, nil) + } + + if resultSetErr != nil { + logger.Errorf("failed to set the result data: %s", err) + return C.int64_t(0) } - enc, _ := scale.Marshal(result) - ptr, _ := toWasmMemory(instanceContext, enc) + enc, err := scale.Marshal(result) + if err != nil { + logger.Errorf("failed to scale marshal the result: %s", err) + return C.int64_t(0) + } + + ptr, err := toWasmMemory(instanceContext, enc) + if err != nil { + logger.Errorf("failed to allocate result on memory: %s", err) + return C.int64_t(0) + } return C.int64_t(ptr) } diff --git a/lib/runtime/wasmer/imports_test.go b/lib/runtime/wasmer/imports_test.go index a590773c65..ae773df984 100644 --- a/lib/runtime/wasmer/imports_test.go +++ b/lib/runtime/wasmer/imports_test.go @@ -365,19 +365,14 @@ func Test_ext_offchain_http_request_add_header(t *testing.T) { ok, err := gotResult.Unwrap() if tcase.expectedErr { require.Error(t, err) - - offchainReq := inst.ctx.OffchainHTTPSet.Get(reqID) - gotValue := offchainReq.Request.Header.Get(tcase.key) - require.Empty(t, gotValue) - } else { require.NoError(t, err) - - offchainReq := inst.ctx.OffchainHTTPSet.Get(reqID) - gotValue := offchainReq.Request.Header.Get(tcase.key) - require.Equal(t, tcase.value, gotValue) } + offchainReq := inst.ctx.OffchainHTTPSet.Get(reqID) + gotValue := offchainReq.Request.Header.Get(tcase.key) + require.Equal(t, tcase.value, gotValue) + require.Nil(t, ok) }) } From eee9136cb55fe100889762c6afb35ec6eb8cf4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 19 Nov 2021 09:48:10 -0400 Subject: [PATCH 33/37] chore: adjust the error flow --- lib/runtime/wasmer/imports.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index a9e5f15026..7182403c85 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1723,24 +1723,24 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa logger.Debug("executing...") instanceContext := wasm.IntoInstanceContext(context) + runtimeCtx := instanceContext.Data().(*runtime.Context) httpMethod := asMemorySlice(instanceContext, methodSpan) uri := asMemorySlice(instanceContext, uriSpan) result := scale.NewResult(int16(0), nil) - var resultSetErr error - runtimeCtx := instanceContext.Data().(*runtime.Context) reqID, err := runtimeCtx.OffchainHTTPSet.StartRequest(string(httpMethod), string(uri)) - if err != nil { + // StartRequest error already was logged logger.Errorf("failed to start request: %s", err) - resultSetErr = result.Set(scale.Err, nil) + err = result.Set(scale.OK, reqID) } else { - resultSetErr = result.Set(scale.OK, reqID) + err = result.Set(scale.Err, reqID) } - if resultSetErr != nil { + // note: just check if an error occurs while setting the result data + if err != nil { logger.Errorf("failed to set the result data: %s", err) return C.int64_t(0) } @@ -1772,17 +1772,16 @@ func ext_offchain_http_request_add_header_version_1(context unsafe.Pointer, reqI offchainReq := runtimeCtx.OffchainHTTPSet.Get(int16(reqID)) result := scale.NewResult(nil, nil) - var resultSetErr error + resultMode := scale.OK err := offchainReq.AddHeader(string(name), string(value)) if err != nil { logger.Errorf("failed to add request header: %s", err) - resultSetErr = result.Set(scale.Err, nil) - } else { - resultSetErr = result.Set(scale.OK, nil) + resultMode = scale.Err } - if resultSetErr != nil { + err = result.Set(resultMode, nil) + if err != nil { logger.Errorf("failed to set the result data: %s", err) return C.int64_t(0) } From 94100598e9bdb95290322e6978b638416ae915f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 19 Nov 2021 12:18:34 -0400 Subject: [PATCH 34/37] chore: fix result return --- lib/runtime/wasmer/imports.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime/wasmer/imports.go b/lib/runtime/wasmer/imports.go index 7182403c85..4588fe84c9 100644 --- a/lib/runtime/wasmer/imports.go +++ b/lib/runtime/wasmer/imports.go @@ -1734,9 +1734,9 @@ func ext_offchain_http_request_start_version_1(context unsafe.Pointer, methodSpa if err != nil { // StartRequest error already was logged logger.Errorf("failed to start request: %s", err) - err = result.Set(scale.OK, reqID) + err = result.Set(scale.Err, nil) } else { - err = result.Set(scale.Err, reqID) + err = result.Set(scale.OK, reqID) } // note: just check if an error occurs while setting the result data From a4441a2796a6ae3e674949bdd6496c3161dc5db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 19 Nov 2021 15:01:20 -0400 Subject: [PATCH 35/37] chore: update the host runtime link --- lib/runtime/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime/constants.go b/lib/runtime/constants.go index 4c91fd4189..f2b7d9cb7d 100644 --- a/lib/runtime/constants.go +++ b/lib/runtime/constants.go @@ -34,7 +34,7 @@ const ( // v0.9 test API wasm HOST_API_TEST_RUNTIME = "hostapi_runtime" HOST_API_TEST_RUNTIME_FP = "hostapi_runtime.compact.wasm" - HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/7edb16fa9df0b93e59489bf85d958a564d2787ec/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" + HOST_API_TEST_RUNTIME_URL = "https://github.com/ChainSafe/polkadot-spec/blob/4d190603d21d4431888bcb1ec546c4dc03b7bf93/test/runtimes/hostapi/hostapi_runtime.compact.wasm?raw=true" // v0.8 substrate runtime with modified name and babe C=(1, 1) DEV_RUNTIME = "dev_runtime" From 92b9f4954079b558d03a5cd021c2e6a35f7d4e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 19 Nov 2021 20:29:10 -0400 Subject: [PATCH 36/37] chore: use request context to store bool values --- README.md | 2 ++ lib/runtime/offchain/httpset.go | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9db92e7010..3cc74233df 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,8 @@ build gossamer command: make gossamer ``` + + ### Run Development Node To initialise a development node: diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index e687c1f4e9..73c64e3398 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -4,6 +4,7 @@ package offchain import ( + "context" "errors" "fmt" "net/http" @@ -11,6 +12,13 @@ import ( "sync" ) +type contextKey string + +const ( + waitingKey contextKey = "waiting" + invalidKey contextKey = "invalid" +) + const maxConcurrentRequests = 1000 var ( @@ -56,17 +64,19 @@ func (b requestIDBuffer) put(i int16) error { // Request holds the request object and update the invalid and waiting status whenever // the request starts or is waiting to be read type Request struct { - Request *http.Request - invalid, waiting bool + Request *http.Request } // AddHeader adds a new HTTP header into request property, only if request is valid and has not started yet func (r *Request) AddHeader(name, value string) error { - if r.invalid { + invalid := r.Request.Context().Value(invalidKey).(bool) + waiting := r.Request.Context().Value(waitingKey).(bool) + + if invalid { return errRequestInvalid } - if r.waiting { + if waiting { return errRequestAlreadyStarted } @@ -112,6 +122,12 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { } req, err := http.NewRequest(method, uri, nil) + + ctx := context.WithValue(req.Context(), waitingKey, false) + ctx = context.WithValue(ctx, invalidKey, false) + + req = req.WithContext(ctx) + if err != nil { return 0, err } From 75af512805386686c0c35d04145f76a838296f82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 19 Nov 2021 21:24:51 -0400 Subject: [PATCH 37/37] chore: fix the lint issues --- lib/runtime/offchain/httpset.go | 14 ++++---------- lib/runtime/offchain/httpset_test.go | 11 ++++++----- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/runtime/offchain/httpset.go b/lib/runtime/offchain/httpset.go index 73c64e3398..eb47059709 100644 --- a/lib/runtime/offchain/httpset.go +++ b/lib/runtime/offchain/httpset.go @@ -26,7 +26,6 @@ var ( errIntBufferFull = errors.New("int buffer is full") errRequestIDNotAvailable = errors.New("request id not available") errRequestInvalid = errors.New("request is invalid") - errRequestAlreadyStarted = errors.New("request has already started") errInvalidHeaderKey = errors.New("invalid header key") ) @@ -67,19 +66,13 @@ type Request struct { Request *http.Request } -// AddHeader adds a new HTTP header into request property, only if request is valid and has not started yet +// AddHeader adds a new HTTP header into request property, only if request is valid func (r *Request) AddHeader(name, value string) error { - invalid := r.Request.Context().Value(invalidKey).(bool) - waiting := r.Request.Context().Value(waitingKey).(bool) - - if invalid { + invalid, ok := r.Request.Context().Value(invalidKey).(bool) + if ok && invalid { return errRequestInvalid } - if waiting { - return errRequestAlreadyStarted - } - name = strings.TrimSpace(name) if len(name) == 0 { return fmt.Errorf("%w: empty header key", errInvalidHeaderKey) @@ -122,6 +115,7 @@ func (p *HTTPSet) StartRequest(method, uri string) (int16, error) { } req, err := http.NewRequest(method, uri, nil) + req.Header = make(http.Header) ctx := context.WithValue(req.Context(), waitingKey, false) ctx = context.WithValue(ctx, invalidKey, false) diff --git a/lib/runtime/offchain/httpset_test.go b/lib/runtime/offchain/httpset_test.go index 1698666a46..d9b40c8bf2 100644 --- a/lib/runtime/offchain/httpset_test.go +++ b/lib/runtime/offchain/httpset_test.go @@ -4,6 +4,7 @@ package offchain import ( + "context" "fmt" "net/http" "testing" @@ -53,19 +54,19 @@ func TestHTTPSetGet(t *testing.T) { func TestOffchainRequest_AddHeader(t *testing.T) { t.Parallel() + invalidCtx := context.WithValue(context.Background(), invalidKey, true) + invalidReq, err := http.NewRequestWithContext(invalidCtx, http.MethodGet, "http://test.com", nil) + require.NoError(t, err) + cases := map[string]struct { offReq Request err error headerK, headerV string }{ "should return invalid request": { - offReq: Request{invalid: true}, + offReq: Request{invalidReq}, err: errRequestInvalid, }, - "should return request already started": { - offReq: Request{waiting: true}, - err: errRequestAlreadyStarted, - }, "should add header": { offReq: Request{Request: &http.Request{Header: make(http.Header)}}, headerK: "key",