From 7dce18d0ea04f1ac60620adf03a7c07f55902f2d Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Wed, 1 Sep 2021 20:59:18 +0300 Subject: [PATCH 01/43] wasm-edge dep --- .gitmodules | 3 +++ CMakeLists.txt | 2 ++ deps/wasmedge | 1 + 3 files changed, 6 insertions(+) create mode 100644 .gitmodules create mode 160000 deps/wasmedge diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..f483ac1f11 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "deps/wasmedge"] + path = deps/wasmedge + url = git@github.com:WasmEdge/WasmEdge.git diff --git a/CMakeLists.txt b/CMakeLists.txt index f781c8f5f9..9c1426f376 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,3 +178,5 @@ if(TESTING) endif() add_subdirectory(node) + +add_subdirectory(deps/wasmedge) diff --git a/deps/wasmedge b/deps/wasmedge new file mode 160000 index 0000000000..24aa254fa0 --- /dev/null +++ b/deps/wasmedge @@ -0,0 +1 @@ +Subproject commit 24aa254fa079b747239728f804a6af8034289b0c From 96cb8915dd3096da219f7a6c041297a800143616 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Thu, 2 Sep 2021 09:19:37 +0300 Subject: [PATCH 02/43] wasmedge host api --- core/host_api/impl/host_api_impl.cpp | 83 ++++++++++++++++++ core/host_api/impl/host_api_impl.hpp | 4 + core/host_api/impl/wasmedge/reg.h | 120 +++++++++++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 core/host_api/impl/wasmedge/reg.h diff --git a/core/host_api/impl/host_api_impl.cpp b/core/host_api/impl/host_api_impl.cpp index 3a4558b124..998c64de97 100644 --- a/core/host_api/impl/host_api_impl.cpp +++ b/core/host_api/impl/host_api_impl.cpp @@ -14,6 +14,7 @@ #include "crypto/sr25519/sr25519_provider_impl.hpp" #include "host_api/impl/offchain_extension.hpp" #include "runtime/trie_storage_provider.hpp" +#include "host_api/impl/wasmedge/reg.h" namespace kagome::host_api { @@ -62,6 +63,88 @@ namespace kagome::host_api { storage_ext_.reset(); } + void HostApiImpl::reg(WasmEdge_ImportObjectContext *ImpObj); + { + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_trie_blake2_256_ordered_root_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_index_set_version_1, ImpObj); + + REGISTER_HOST_API_FUNC(HostApiImpl, ext_logging_log_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_ed25519_generate_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_ed25519_verify_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_finish_batch_verify_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_secp256k1_ecdsa_recover_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApi, + ext_crypto_secp256k1_ecdsa_recover_compressed_version_1, + ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_sr25519_generate_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_sr25519_public_keys_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_sr25519_sign_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_sr25519_verify_version_2, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_start_batch_verify_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_hashing_blake2_128_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_hashing_blake2_256_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_hashing_keccak_256_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_twox_128_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_twox_64_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_free_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_malloc_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_hex_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_utf8_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_misc_runtime_version_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_is_validator_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_local_storage_clear_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApi, ext_offchain_local_storage_compare_and_set_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_local_storage_get_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_local_storage_set_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_network_state_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_random_seed_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_submit_transaction_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_timestamp_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_append_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_changes_root_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_clear_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_clear_prefix_version_2, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_commit_transaction_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_exists_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_get_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_next_key_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_read_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_rollback_transaction_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_root_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_set_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_start_transaction_version_1, ImpObj); + } + runtime::WasmSpan HostApiImpl::ext_storage_read_version_1( runtime::WasmSpan key, runtime::WasmSpan value_out, diff --git a/core/host_api/impl/host_api_impl.hpp b/core/host_api/impl/host_api_impl.hpp index cf8971f8e1..3cd55fd2dc 100644 --- a/core/host_api/impl/host_api_impl.hpp +++ b/core/host_api/impl/host_api_impl.hpp @@ -17,6 +17,8 @@ #include "host_api/impl/child_storage_extension.hpp" #include "offchain/impl/offchain_persistent_storage.hpp" +struct WasmEdge_ImportObjectContext; + namespace kagome::runtime { class Core; class MemoryProvider; @@ -50,6 +52,8 @@ namespace kagome::host_api { ~HostApiImpl() override = default; + void reg(WasmEdge_ImportObjectContext* ImpObj); + void reset() override; // ------------------------ Storage extensions v1 ------------------------ diff --git a/core/host_api/impl/wasmedge/reg.h b/core/host_api/impl/wasmedge/reg.h new file mode 100644 index 0000000000..0a1ac58229 --- /dev/null +++ b/core/host_api/impl/wasmedge/reg.h @@ -0,0 +1,120 @@ +#pragma once + +#include + +#include +#include +#include + +template WasmEdge_Value (*fromType())(const T) { return nullptr; } +template <> WasmEdge_Value (*fromType())(const int32_t) { + return WasmEdge_ValueGenI32; +} +template <> WasmEdge_Value (*fromType())(const int64_t) { + return WasmEdge_ValueGenI64; +} +template <> WasmEdge_Value (*fromType())(const float) { + return WasmEdge_ValueGenF32; +} +template <> WasmEdge_Value (*fromType())(const double) { + return WasmEdge_ValueGenF64; +} +template <> WasmEdge_Value (*fromType<__int128>())(const __int128) { + return WasmEdge_ValueGenV128; +} + +template T (*toType())(const WasmEdge_Value) { return nullptr; } +template <> int32_t (*toType())(const WasmEdge_Value) { + return WasmEdge_ValueGetI32; +} +template <> int64_t (*toType())(const WasmEdge_Value) { + return WasmEdge_ValueGetI64; +} +template <> float (*toType())(const WasmEdge_Value) { + return WasmEdge_ValueGetF32; +} +template <> double (*toType())(const WasmEdge_Value) { + return WasmEdge_ValueGetF64; +} +template <> __int128 (*toType<__int128>())(const WasmEdge_Value) { + return WasmEdge_ValueGetV128; +} + +template WasmEdge_ValType type() { return WasmEdge_ValType_V128; } +template <> WasmEdge_ValType type() { return WasmEdge_ValType_I32; } +template <> WasmEdge_ValType type() { return WasmEdge_ValType_I64; } +template <> WasmEdge_ValType type() { return WasmEdge_ValType_F32; } +template <> WasmEdge_ValType type() { return WasmEdge_ValType_F64; } + +template +auto internal_fun(T *fun, const WasmEdge_Value *In, const TP &tup, + std::index_sequence) { + return (fun->*f)(std::get(tup)(In[I])...); +} + +template struct HostApiFuncRet; +template +struct HostApiFuncRet { + using Ret = R; +}; + +/* Host function body definition. */ +template +WasmEdge_Result fun(void *Data, WasmEdge_MemoryInstanceContext *MemCxt, + const WasmEdge_Value *In, WasmEdge_Value *Out) { + auto indices = std::index_sequence_for{}; + auto tup = std::make_tuple(toType()...); + if constexpr (not std::is_same_v< + void, typename HostApiFuncRet::Ret>) { + auto res = internal_fun(static_cast(Data), In, tup, indices); + Out[0] = fromType()(res); + } + else { + internal_fun(static_cast(Data), In, tup, indices); + } + return WasmEdge_Result_Success; +} + +template struct HostApiFunc; +template +struct HostApiFunc { + using Ret = R; + static WasmEdge_Result (*fun())(void *Data, + WasmEdge_MemoryInstanceContext *MemCxt, + const WasmEdge_Value *In, + WasmEdge_Value *Out) { + return ::fun; + } + + static WasmEdge_FunctionTypeContext *type() { + enum WasmEdge_ValType ParamList[sizeof...(Args)] = {::type()...}; + + size_t r = 1; + enum WasmEdge_ValType ReturnListNonVoid[1] = {::type()}; + enum WasmEdge_ValType *ReturnList = ReturnListNonVoid; + + if constexpr (std::is_same_v) { + r = 0; + ReturnList = nullptr; + } + + return WasmEdge_FunctionTypeCreate(ParamList, sizeof...(Args), ReturnList, + r); + } +}; + +template +void registerHostApiFunc(const std::string &name, + WasmEdge_ImportObjectContext *ImpObj) { + WasmEdge_FunctionTypeContext *HostFType = + HostApiFunc::type(); + WasmEdge_HostFunctionContext *HostFunc = WasmEdge_HostFunctionCreate( + HostFType, HostApiFunc::fun(), 0); + WasmEdge_FunctionTypeDelete(HostFType); + WasmEdge_String HostFuncName = WasmEdge_StringCreateByCString(name.c_str()); + WasmEdge_ImportObjectAddHostFunction(ImpObj, HostFuncName, HostFunc); + WasmEdge_StringDelete(HostFuncName); +} + +#define REGISTER_HOST_API_FUNC(class, name, obj) \ + registerHostApiFunc<&class ::name>(#name, obj) From 72580e0c9cc06a2cecd6ae77477db44379261b4e Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Thu, 2 Sep 2021 22:24:50 +0300 Subject: [PATCH 03/43] wasmedge compile host api --- .gitmodules | 3 - CMakeLists.txt | 5 +- conanfile.txt | 10 ++ core/host_api/CMakeLists.txt | 1 + core/host_api/impl/host_api_impl.cpp | 5 +- core/host_api/impl/host_api_impl.hpp | 33 +++++- core/host_api/impl/wasmedge/reg.h | 146 +++++++++++++++++++++------ deps/wasmedge | 1 - 8 files changed, 163 insertions(+), 41 deletions(-) delete mode 100644 .gitmodules create mode 100644 conanfile.txt delete mode 160000 deps/wasmedge diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index f483ac1f11..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "deps/wasmedge"] - path = deps/wasmedge - url = git@github.com:WasmEdge/WasmEdge.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c1426f376..f51c114c0c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,9 @@ include("cmake/Hunter/init.cmake") project(kagome C CXX) +include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) +conan_define_targets() + include(cmake/print.cmake) print("C flags: ${CMAKE_C_FLAGS}") print("CXX flags: ${CMAKE_CXX_FLAGS}") @@ -178,5 +181,3 @@ if(TESTING) endif() add_subdirectory(node) - -add_subdirectory(deps/wasmedge) diff --git a/conanfile.txt b/conanfile.txt new file mode 100644 index 0000000000..98d0113d2e --- /dev/null +++ b/conanfile.txt @@ -0,0 +1,10 @@ +[generators] +cmake_find_package_multi + +[requires] +WasmEdge/0.8.1@sanblch/stable + +[options] +rocksdb:with_snappy=True + +[imports] diff --git a/core/host_api/CMakeLists.txt b/core/host_api/CMakeLists.txt index 61f97f3f22..7e17ac3df6 100644 --- a/core/host_api/CMakeLists.txt +++ b/core/host_api/CMakeLists.txt @@ -16,5 +16,6 @@ target_link_libraries(host_api storage_extension child_storage_extension offchain_extension + CONAN_PKG::WasmEdge ) kagome_install(host_api) diff --git a/core/host_api/impl/host_api_impl.cpp b/core/host_api/impl/host_api_impl.cpp index 998c64de97..1e01d19457 100644 --- a/core/host_api/impl/host_api_impl.cpp +++ b/core/host_api/impl/host_api_impl.cpp @@ -63,8 +63,7 @@ namespace kagome::host_api { storage_ext_.reset(); } - void HostApiImpl::reg(WasmEdge_ImportObjectContext *ImpObj); - { + void HostApiImpl::reg(WasmEdge_ImportObjectContext *ImpObj) { REGISTER_HOST_API_FUNC( HostApiImpl, ext_trie_blake2_256_ordered_root_version_1, ImpObj); REGISTER_HOST_API_FUNC( @@ -112,7 +111,7 @@ namespace kagome::host_api { REGISTER_HOST_API_FUNC( HostApiImpl, ext_offchain_local_storage_clear_version_1, ImpObj); REGISTER_HOST_API_FUNC( - HostApi, ext_offchain_local_storage_compare_and_set_version_1, ImpObj); + HostApiImpl, ext_offchain_local_storage_compare_and_set_version_1, ImpObj); REGISTER_HOST_API_FUNC( HostApiImpl, ext_offchain_local_storage_get_version_1, ImpObj); REGISTER_HOST_API_FUNC( diff --git a/core/host_api/impl/host_api_impl.hpp b/core/host_api/impl/host_api_impl.hpp index 3cd55fd2dc..9ebc954cfd 100644 --- a/core/host_api/impl/host_api_impl.hpp +++ b/core/host_api/impl/host_api_impl.hpp @@ -25,6 +25,9 @@ namespace kagome::runtime { class CoreApiFactory; } // namespace kagome::runtime +using I32 = int32_t; +using I64 = int64_t; + namespace kagome::host_api { class OffchainExtension; @@ -52,10 +55,38 @@ namespace kagome::host_api { ~HostApiImpl() override = default; - void reg(WasmEdge_ImportObjectContext* ImpObj); + void reg(WasmEdge_ImportObjectContext *ImpObj); void reset() override; + void ext_offchain_index_set_version_1(I64, I64) {} + I32 ext_offchain_is_validator_version_1() { + return 0; + } + void ext_offchain_local_storage_clear_version_1(I32, I64) {} + I32 ext_offchain_local_storage_compare_and_set_version_1(I32, + I64, + I64, + I64) { + return 0; + } + I64 ext_offchain_local_storage_get_version_1(I32, I64) { + return 0; + } + void ext_offchain_local_storage_set_version_1(I32, I64, I64) {} + I64 ext_offchain_network_state_version_1() { + return 0; + } + I32 ext_offchain_random_seed_version_1() { + return 0; + } + I64 ext_offchain_submit_transaction_version_1(I64) { + return 0; + } + I64 ext_offchain_timestamp_version_1() { + return 0; + } + // ------------------------ Storage extensions v1 ------------------------ runtime::WasmSpan ext_storage_read_version_1( diff --git a/core/host_api/impl/wasmedge/reg.h b/core/host_api/impl/wasmedge/reg.h index 0a1ac58229..154e1ba942 100644 --- a/core/host_api/impl/wasmedge/reg.h +++ b/core/host_api/impl/wasmedge/reg.h @@ -6,76 +6,128 @@ #include #include -template WasmEdge_Value (*fromType())(const T) { return nullptr; } -template <> WasmEdge_Value (*fromType())(const int32_t) { +template +WasmEdge_Value (*fromType())(const T) { + return nullptr; +} +template <> +WasmEdge_Value (*fromType())(const int32_t) { return WasmEdge_ValueGenI32; } -template <> WasmEdge_Value (*fromType())(const int64_t) { +template <> +WasmEdge_Value (*fromType())(const int64_t) { return WasmEdge_ValueGenI64; } -template <> WasmEdge_Value (*fromType())(const float) { +template <> +WasmEdge_Value (*fromType())(const float) { return WasmEdge_ValueGenF32; } -template <> WasmEdge_Value (*fromType())(const double) { +template <> +WasmEdge_Value (*fromType())(const double) { return WasmEdge_ValueGenF64; } -template <> WasmEdge_Value (*fromType<__int128>())(const __int128) { +template <> +WasmEdge_Value (*fromType<__int128>())(const __int128) { return WasmEdge_ValueGenV128; } -template T (*toType())(const WasmEdge_Value) { return nullptr; } -template <> int32_t (*toType())(const WasmEdge_Value) { +template +T (*toType()) +(const WasmEdge_Value) { + return nullptr; +} +template <> +int32_t (*toType())(const WasmEdge_Value) { return WasmEdge_ValueGetI32; } -template <> int64_t (*toType())(const WasmEdge_Value) { +template <> +int64_t (*toType())(const WasmEdge_Value) { return WasmEdge_ValueGetI64; } -template <> float (*toType())(const WasmEdge_Value) { +template <> +float (*toType())(const WasmEdge_Value) { return WasmEdge_ValueGetF32; } -template <> double (*toType())(const WasmEdge_Value) { +template <> +double (*toType())(const WasmEdge_Value) { return WasmEdge_ValueGetF64; } -template <> __int128 (*toType<__int128>())(const WasmEdge_Value) { +template <> +__int128 (*toType<__int128>())(const WasmEdge_Value) { return WasmEdge_ValueGetV128; } -template WasmEdge_ValType type() { return WasmEdge_ValType_V128; } -template <> WasmEdge_ValType type() { return WasmEdge_ValType_I32; } -template <> WasmEdge_ValType type() { return WasmEdge_ValType_I64; } -template <> WasmEdge_ValType type() { return WasmEdge_ValType_F32; } -template <> WasmEdge_ValType type() { return WasmEdge_ValType_F64; } +template +struct MakeSigned { + using type = typename std:: + conditional_t, typename std::make_signed_t, T>; +}; +template<> +struct MakeSigned { + using type = void; +}; + +template +WasmEdge_ValType type() { + return WasmEdge_ValType_V128; +} +template <> +WasmEdge_ValType type() { + return WasmEdge_ValType_I32; +} +template <> +WasmEdge_ValType type() { + return WasmEdge_ValType_I64; +} +template <> +WasmEdge_ValType type() { + return WasmEdge_ValType_F32; +} +template <> +WasmEdge_ValType type() { + return WasmEdge_ValType_F64; +} template -auto internal_fun(T *fun, const WasmEdge_Value *In, const TP &tup, +auto internal_fun(T *fun, + const WasmEdge_Value *In, + const TP &tup, std::index_sequence) { return (fun->*f)(std::get(tup)(In[I])...); } -template struct HostApiFuncRet; +template +struct HostApiFuncRet; template struct HostApiFuncRet { using Ret = R; }; +template +struct HostApiFuncRet { + using Ret = R; +}; /* Host function body definition. */ template -WasmEdge_Result fun(void *Data, WasmEdge_MemoryInstanceContext *MemCxt, - const WasmEdge_Value *In, WasmEdge_Value *Out) { +WasmEdge_Result fun(void *Data, + WasmEdge_MemoryInstanceContext *MemCxt, + const WasmEdge_Value *In, + WasmEdge_Value *Out) { auto indices = std::index_sequence_for{}; - auto tup = std::make_tuple(toType()...); + auto tup = std::make_tuple(toType::type>()...); if constexpr (not std::is_same_v< - void, typename HostApiFuncRet::Ret>) { + void, + typename HostApiFuncRet::Ret>) { auto res = internal_fun(static_cast(Data), In, tup, indices); Out[0] = fromType()(res); - } - else { + } else { internal_fun(static_cast(Data), In, tup, indices); } return WasmEdge_Result_Success; } -template struct HostApiFunc; +template +struct HostApiFunc; template struct HostApiFunc { using Ret = R; @@ -87,10 +139,12 @@ struct HostApiFunc { } static WasmEdge_FunctionTypeContext *type() { - enum WasmEdge_ValType ParamList[sizeof...(Args)] = {::type()...}; + enum WasmEdge_ValType ParamList[sizeof...(Args)] = { + ::type::type>()...}; size_t r = 1; - enum WasmEdge_ValType ReturnListNonVoid[1] = {::type()}; + enum WasmEdge_ValType ReturnListNonVoid[1] = { + ::type::type>()}; enum WasmEdge_ValType *ReturnList = ReturnListNonVoid; if constexpr (std::is_same_v) { @@ -98,11 +152,41 @@ struct HostApiFunc { ReturnList = nullptr; } - return WasmEdge_FunctionTypeCreate(ParamList, sizeof...(Args), ReturnList, - r); + return WasmEdge_FunctionTypeCreate( + ParamList, sizeof...(Args), ReturnList, r); } }; +template +struct HostApiFunc { + using Ret = R; + static WasmEdge_Result (*fun())(void *Data, + WasmEdge_MemoryInstanceContext *MemCxt, + const WasmEdge_Value *In, + WasmEdge_Value *Out) { + return ::fun; + } + + static WasmEdge_FunctionTypeContext *type() { + enum WasmEdge_ValType ParamList[sizeof...(Args)] = { + ::type::type>()...}; + + size_t r = 1; + enum WasmEdge_ValType ReturnListNonVoid[1] = { + ::type::type>()}; + enum WasmEdge_ValType *ReturnList = ReturnListNonVoid; + + if constexpr (std::is_same_v) { + r = 0; + ReturnList = nullptr; + } + + return WasmEdge_FunctionTypeCreate( + ParamList, sizeof...(Args), ReturnList, r); + } +}; + + template void registerHostApiFunc(const std::string &name, WasmEdge_ImportObjectContext *ImpObj) { @@ -116,5 +200,5 @@ void registerHostApiFunc(const std::string &name, WasmEdge_StringDelete(HostFuncName); } -#define REGISTER_HOST_API_FUNC(class, name, obj) \ +#define REGISTER_HOST_API_FUNC(class, name, obj) \ registerHostApiFunc<&class ::name>(#name, obj) diff --git a/deps/wasmedge b/deps/wasmedge deleted file mode 160000 index 24aa254fa0..0000000000 --- a/deps/wasmedge +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 24aa254fa079b747239728f804a6af8034289b0c From 0a8981b9898f9f623d8de00a5ed57092360e4c66 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Sat, 4 Sep 2021 11:41:24 +0300 Subject: [PATCH 04/43] wasmedge memory --- core/runtime/wasmedge/CMakeLists.txt | 15 ++++ core/runtime/wasmedge/memory_impl.cpp | 121 ++++++++++++++++++++++++++ core/runtime/wasmedge/memory_impl.hpp | 84 ++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 core/runtime/wasmedge/CMakeLists.txt create mode 100644 core/runtime/wasmedge/memory_impl.cpp create mode 100644 core/runtime/wasmedge/memory_impl.hpp diff --git a/core/runtime/wasmedge/CMakeLists.txt b/core/runtime/wasmedge/CMakeLists.txt new file mode 100644 index 0000000000..4e27c4c758 --- /dev/null +++ b/core/runtime/wasmedge/CMakeLists.txt @@ -0,0 +1,15 @@ +# +# Copyright Soramitsu Co., Ltd. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +add_library(wasmedge_wasm_memory + memory_impl.cpp + ) +target_link_libraries(wasmedge_wasm_memory + buffer + logger + CONAN_PKG::WasmEdge + memory_allocator + ) +kagome_install(wasmedge_wasm_memory) diff --git a/core/runtime/wasmedge/memory_impl.cpp b/core/runtime/wasmedge/memory_impl.cpp new file mode 100644 index 0000000000..9246d446a6 --- /dev/null +++ b/core/runtime/wasmedge/memory_impl.cpp @@ -0,0 +1,121 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wasmedge/memory_impl.hpp" + +#include "runtime/common/memory_allocator.hpp" +#include "runtime/ptr_size.hpp" + +#include + +namespace { + template + auto toArray(T t) { + std::array res; + for (unsigned i = 0; i < N; ++i) { + res[i] = static_cast((t >> (N - i - 1) * 8) & 0xFF); + } + return res; + } +} // namespace + +namespace kagome::runtime::wasmedge { + + MemoryImpl::MemoryImpl(wasm::WasmEdge_MemoryInstanceContext *memory, + std::unique_ptr &&allocator) + : memory_{memory}, + size_{kInitialMemorySize}, + allocator_{std::move(allocator)}, + logger_{log::createLogger("Binaryen Memory", "binaryen")} { + resize(size_); + } + + MemoryImpl::MemoryImpl(wasm::ShellExternalInterface::Memory *memory, + WasmSize heap_base) + : MemoryImpl{memory, + std::make_unique( + MemoryAllocator::MemoryHandle{ + [this](auto new_size) { return resize(new_size); }, + [this]() { return size_; }}, + kInitialMemorySize, + heap_base)} {} + + WasmPointer MemoryImpl::allocate(WasmSize size) { + return allocator_->allocate(size); + } + + boost::optional MemoryImpl::deallocate(WasmPointer ptr) { + return allocator_->deallocate(ptr); + } + + common::Buffer MemoryImpl::loadN(kagome::runtime::WasmPointer addr, + kagome::runtime::WasmSize n) const { + BOOST_ASSERT(size_ > addr and size_ - addr >= n); + common::Buffer res(n, 0); + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, n); + return res; + } + + std::string MemoryImpl::loadStr(kagome::runtime::WasmPointer addr, + kagome::runtime::WasmSize length) const { + BOOST_ASSERT(size_ > addr and size_ - addr >= length); + std::string res(length, '\0'); + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, length); + return res; + } + + void MemoryImpl::store8(WasmPointer addr, int8_t value) { + BOOST_ASSERT((allocator_->checkAddress(addr))); + WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + memory_, toArray(value).data(), addr, sizeof(value)); + } + + void MemoryImpl::store16(WasmPointer addr, int16_t value) { + BOOST_ASSERT((allocator_->checkAddress(addr))); + WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + memory_, toArray(value).data(), addr, sizeof(value)); + } + + void MemoryImpl::store32(WasmPointer addr, int32_t value) { + BOOST_ASSERT((allocator_->checkAddress(addr))); + WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + memory_, toArray(value).data(), addr, sizeof(value)); + } + + void MemoryImpl::store64(WasmPointer addr, int64_t value) { + BOOST_ASSERT((allocator_->checkAddress(addr))); + WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + memory_, toArray(value).data(), addr, sizeof(value)); + } + + void MemoryImpl::store128(WasmPointer addr, + const std::array &value) { + BOOST_ASSERT((allocator_->checkAddress>(addr))); + WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + memory_, value.data(), addr, sizeof(value)); + } + + void MemoryImpl::storeBuffer(kagome::runtime::WasmPointer addr, + gsl::span value) { + const auto size = static_cast(value.size()); + BOOST_ASSERT((allocator_->checkAddress(addr, size))); + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, value.data(), addr, size); + } + + WasmSpan MemoryImpl::storeBuffer(gsl::span value) { + const auto size = static_cast(value.size()); + BOOST_ASSERT(std::numeric_limits::max() > size); + auto wasm_pointer = allocate(size); + if (wasm_pointer == 0) { + return 0; + } + storeBuffer(wasm_pointer, value); + return PtrSize(wasm_pointer, value.size()).combine(); + } + +} // namespace kagome::runtime::binaryen diff --git a/core/runtime/wasmedge/memory_impl.hpp b/core/runtime/wasmedge/memory_impl.hpp new file mode 100644 index 0000000000..3c6a959549 --- /dev/null +++ b/core/runtime/wasmedge/memory_impl.hpp @@ -0,0 +1,84 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_RUNTIME_BINARYEN_WASM_MEMORY_IMPL_HPP +#define KAGOME_RUNTIME_BINARYEN_WASM_MEMORY_IMPL_HPP + +#include + +#include +#include // for std::memset in gcc +#include +#include + +#include + +#include "common/literals.hpp" +#include "log/logger.hpp" +#include "primitives/math.hpp" +#include "runtime/memory.hpp" + +struct WasmEdge_MemoryInstanceContext; + +namespace kagome::runtime { + class MemoryAllocator; +} + +namespace kagome::runtime::wasmedge { + + class MemoryImpl final : public Memory { + public: + MemoryImpl(WasmEdge_MemoryInstanceContext *memory, + std::unique_ptr&& allocator); + MemoryImpl(const MemoryImpl ©) = delete; + MemoryImpl &operator=(const MemoryImpl ©) = delete; + MemoryImpl(MemoryImpl &&move) = delete; + MemoryImpl &operator=(MemoryImpl &&move) = delete; + ~MemoryImpl() override = default; + + WasmPointer allocate(WasmSize size) override; + boost::optional deallocate(WasmPointer ptr) override; + + common::Buffer loadN(kagome::runtime::WasmPointer addr, + kagome::runtime::WasmSize n) const override; + std::string loadStr(kagome::runtime::WasmPointer addr, + kagome::runtime::WasmSize length) const override; + + void store8(WasmPointer addr, int8_t value) override; + void store16(WasmPointer addr, int16_t value) override; + void store32(WasmPointer addr, int32_t value) override; + void store64(WasmPointer addr, int64_t value) override; + void store128(WasmPointer addr, + const std::array &value) override; + void storeBuffer(kagome::runtime::WasmPointer addr, + gsl::span value) override; + + WasmSpan storeBuffer(gsl::span value) override; + + void resize(WasmSize new_size) override { + /** + * We use this condition to avoid + * deallocated_ pointers fixup + */ + if (new_size >= size_) { + size_ = new_size; + memory_->resize(new_size); + } + } + + WasmSize size() const override { + return size_; + } + + private: + WasmEdge_ImportObjectContext *memory_; + WasmSize size_; + std::unique_ptr allocator_; + + log::Logger logger_; + }; +} // namespace kagome::runtime::wasmedge + +#endif // KAGOME_RUNTIME_WASMEDGE_WASM_MEMORY_IMPL_HPP From d93164848ca3c313ea8ab75d18b9e6e0e3e6ae3a Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Tue, 7 Sep 2021 04:11:50 +0300 Subject: [PATCH 05/43] fix wasmedge memory --- conanfile.txt | 1 + core/runtime/CMakeLists.txt | 1 + core/runtime/common/memory_allocator.cpp | 5 ++ core/runtime/common/memory_allocator.hpp | 1 + core/runtime/wasmedge/CMakeLists.txt | 2 + core/runtime/wasmedge/memory_factory.cpp | 17 ++++ core/runtime/wasmedge/memory_factory.hpp | 23 ++++++ core/runtime/wasmedge/memory_impl.cpp | 95 ++++++++++++++++++++--- core/runtime/wasmedge/memory_impl.hpp | 31 ++++---- core/runtime/wasmedge/memory_provider.cpp | 43 ++++++++++ core/runtime/wasmedge/memory_provider.hpp | 37 +++++++++ 11 files changed, 230 insertions(+), 26 deletions(-) create mode 100644 core/runtime/wasmedge/memory_factory.cpp create mode 100644 core/runtime/wasmedge/memory_factory.hpp create mode 100644 core/runtime/wasmedge/memory_provider.cpp create mode 100644 core/runtime/wasmedge/memory_provider.hpp diff --git a/conanfile.txt b/conanfile.txt index 98d0113d2e..e26a53ab8a 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -1,4 +1,5 @@ [generators] +cmake cmake_find_package_multi [requires] diff --git a/core/runtime/CMakeLists.txt b/core/runtime/CMakeLists.txt index 340fc539d7..93058aae0e 100644 --- a/core/runtime/CMakeLists.txt +++ b/core/runtime/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(common) add_subdirectory(binaryen) +add_subdirectory(wasmedge) add_subdirectory(wavm) add_library(executor INTERFACE) diff --git a/core/runtime/common/memory_allocator.cpp b/core/runtime/common/memory_allocator.cpp index 0f181654c7..a45e25c52d 100644 --- a/core/runtime/common/memory_allocator.cpp +++ b/core/runtime/common/memory_allocator.cpp @@ -33,6 +33,11 @@ namespace kagome::runtime { BOOST_ASSERT(offset_ <= Memory::kMaxMemorySize - size_); } + void MemoryAllocator::reset() { + allocated_.clear(); + deallocated_.clear(); + } + WasmPointer MemoryAllocator::allocate(WasmSize size) { if (size == 0) { return 0; diff --git a/core/runtime/common/memory_allocator.hpp b/core/runtime/common/memory_allocator.hpp index 9af046949a..915ad1990b 100644 --- a/core/runtime/common/memory_allocator.hpp +++ b/core/runtime/common/memory_allocator.hpp @@ -67,6 +67,7 @@ namespace kagome::runtime { std::optional getAllocatedChunkSize(WasmPointer ptr) const; size_t getAllocatedChunksNum() const; size_t getDeallocatedChunksNum() const; + void reset(); private: /** diff --git a/core/runtime/wasmedge/CMakeLists.txt b/core/runtime/wasmedge/CMakeLists.txt index 4e27c4c758..ea1ae244de 100644 --- a/core/runtime/wasmedge/CMakeLists.txt +++ b/core/runtime/wasmedge/CMakeLists.txt @@ -4,7 +4,9 @@ # add_library(wasmedge_wasm_memory + memory_factory.cpp memory_impl.cpp + memory_provider.cpp ) target_link_libraries(wasmedge_wasm_memory buffer diff --git a/core/runtime/wasmedge/memory_factory.cpp b/core/runtime/wasmedge/memory_factory.cpp new file mode 100644 index 0000000000..15fb8e649e --- /dev/null +++ b/core/runtime/wasmedge/memory_factory.cpp @@ -0,0 +1,17 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wasmedge/memory_factory.hpp" + +#include "runtime/common/memory_allocator.hpp" + +namespace kagome::runtime::wasmedge { + + std::unique_ptr WasmedgeMemoryFactory::make( + WasmEdge_MemoryInstanceContext *memory) const { + return std::make_unique(memory); + } + +} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_factory.hpp b/core/runtime/wasmedge/memory_factory.hpp new file mode 100644 index 0000000000..cb341f03a7 --- /dev/null +++ b/core/runtime/wasmedge/memory_factory.hpp @@ -0,0 +1,23 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_WASMEDGE_MEMORY_FACTORY_HPP +#define KAGOME_WASMEDGE_MEMORY_FACTORY_HPP + +#include "runtime/wasmedge/memory_impl.hpp" + +namespace kagome::runtime::wasmedge { + + class WasmedgeMemoryFactory { + public: + virtual ~WasmedgeMemoryFactory() = default; + + std::unique_ptr make( + WasmEdge_MemoryInstanceContext *memory) const; + }; + +} // namespace kagome::runtime::wasmedge + +#endif // KAGOME_WASMEDGE_MEMORY_FACTORY_HPP diff --git a/core/runtime/wasmedge/memory_impl.cpp b/core/runtime/wasmedge/memory_impl.cpp index 9246d446a6..45d74e5903 100644 --- a/core/runtime/wasmedge/memory_impl.cpp +++ b/core/runtime/wasmedge/memory_impl.cpp @@ -5,6 +5,7 @@ #include "runtime/wasmedge/memory_impl.hpp" +#include "common/literals.hpp" #include "runtime/common/memory_allocator.hpp" #include "runtime/ptr_size.hpp" @@ -19,11 +20,22 @@ namespace { } return res; } + + template + auto fromArray(std::array a) { + T res{0}; + for (unsigned i = 0; i < N; ++i) { + res |= static_cast(a[i]) << (i * 8); + } + return res; + } } // namespace namespace kagome::runtime::wasmedge { - MemoryImpl::MemoryImpl(wasm::WasmEdge_MemoryInstanceContext *memory, + using namespace kagome::common::literals; + + MemoryImpl::MemoryImpl(WasmEdge_MemoryInstanceContext *memory, std::unique_ptr &&allocator) : memory_{memory}, size_{kInitialMemorySize}, @@ -32,15 +44,14 @@ namespace kagome::runtime::wasmedge { resize(size_); } - MemoryImpl::MemoryImpl(wasm::ShellExternalInterface::Memory *memory, - WasmSize heap_base) + MemoryImpl::MemoryImpl(WasmEdge_MemoryInstanceContext *memory) : MemoryImpl{memory, std::make_unique( MemoryAllocator::MemoryHandle{ [this](auto new_size) { return resize(new_size); }, [this]() { return size_; }}, kInitialMemorySize, - heap_base)} {} + 5_MB)} {} WasmPointer MemoryImpl::allocate(WasmSize size) { return allocator_->allocate(size); @@ -50,6 +61,70 @@ namespace kagome::runtime::wasmedge { return allocator_->deallocate(ptr); } + int8_t MemoryImpl::load8s(WasmPointer addr) const { + BOOST_ASSERT(allocator_->checkAddress(addr)); + int8_t res; + WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + memory_, reinterpret_cast(&res), addr, sizeof(res)); + return res; + } + uint8_t MemoryImpl::load8u(WasmPointer addr) const { + BOOST_ASSERT(allocator_->checkAddress(addr)); + uint8_t res; + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, &res, addr, sizeof(res)); + return res; + } + int16_t MemoryImpl::load16s(WasmPointer addr) const { + BOOST_ASSERT(allocator_->checkAddress(addr)); + std::array res; + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, sizeof(res)); + return fromArray(res); + } + uint16_t MemoryImpl::load16u(WasmPointer addr) const { + BOOST_ASSERT(allocator_->checkAddress(addr)); + std::array res; + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, sizeof(res)); + return fromArray(res); + } + int32_t MemoryImpl::load32s(WasmPointer addr) const { + BOOST_ASSERT(allocator_->checkAddress(addr)); + std::array res; + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, sizeof(res)); + return fromArray(res); + } + uint32_t MemoryImpl::load32u(WasmPointer addr) const { + BOOST_ASSERT(allocator_->checkAddress(addr)); + std::array res; + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, sizeof(res)); + return fromArray(res); + } + int64_t MemoryImpl::load64s(WasmPointer addr) const { + BOOST_ASSERT(allocator_->checkAddress(addr)); + std::array res; + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, sizeof(res)); + return fromArray(res); + } + uint64_t MemoryImpl::load64u(WasmPointer addr) const { + BOOST_ASSERT(allocator_->checkAddress(addr)); + std::array res; + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, sizeof(res)); + return fromArray(res); + } + std::array MemoryImpl::load128(WasmPointer addr) const { + BOOST_ASSERT((allocator_->checkAddress>(addr))); + std::array res; + WasmEdge_Result Res = + WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, sizeof(res)); + return res; + } + common::Buffer MemoryImpl::loadN(kagome::runtime::WasmPointer addr, kagome::runtime::WasmSize n) const { BOOST_ASSERT(size_ > addr and size_ - addr >= n); @@ -63,8 +138,8 @@ namespace kagome::runtime::wasmedge { kagome::runtime::WasmSize length) const { BOOST_ASSERT(size_ > addr and size_ - addr >= length); std::string res(length, '\0'); - WasmEdge_Result Res = - WasmEdge_MemoryInstanceGetData(memory_, res.data(), addr, length); + WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + memory_, reinterpret_cast(res.data()), addr, length); return res; } @@ -96,15 +171,15 @@ namespace kagome::runtime::wasmedge { const std::array &value) { BOOST_ASSERT((allocator_->checkAddress>(addr))); WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( - memory_, value.data(), addr, sizeof(value)); + memory_, const_cast(value.data()), addr, sizeof(value)); } void MemoryImpl::storeBuffer(kagome::runtime::WasmPointer addr, gsl::span value) { const auto size = static_cast(value.size()); BOOST_ASSERT((allocator_->checkAddress(addr, size))); - WasmEdge_Result Res = - WasmEdge_MemoryInstanceGetData(memory_, value.data(), addr, size); + WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + memory_, const_cast(value.data()), addr, size); } WasmSpan MemoryImpl::storeBuffer(gsl::span value) { @@ -118,4 +193,4 @@ namespace kagome::runtime::wasmedge { return PtrSize(wasm_pointer, value.size()).combine(); } -} // namespace kagome::runtime::binaryen +} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_impl.hpp b/core/runtime/wasmedge/memory_impl.hpp index 3c6a959549..781a049909 100644 --- a/core/runtime/wasmedge/memory_impl.hpp +++ b/core/runtime/wasmedge/memory_impl.hpp @@ -3,10 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef KAGOME_RUNTIME_BINARYEN_WASM_MEMORY_IMPL_HPP -#define KAGOME_RUNTIME_BINARYEN_WASM_MEMORY_IMPL_HPP - -#include +#ifndef KAGOME_RUNTIME_WASMEDGE_MEMORY_IMPL_HPP +#define KAGOME_RUNTIME_WASMEDGE_MEMORY_IMPL_HPP #include #include // for std::memset in gcc @@ -30,6 +28,7 @@ namespace kagome::runtime::wasmedge { class MemoryImpl final : public Memory { public: + MemoryImpl(WasmEdge_MemoryInstanceContext *memory); MemoryImpl(WasmEdge_MemoryInstanceContext *memory, std::unique_ptr&& allocator); MemoryImpl(const MemoryImpl ©) = delete; @@ -41,6 +40,15 @@ namespace kagome::runtime::wasmedge { WasmPointer allocate(WasmSize size) override; boost::optional deallocate(WasmPointer ptr) override; + int8_t load8s(WasmPointer addr) const override; + uint8_t load8u(WasmPointer addr) const override; + int16_t load16s(WasmPointer addr) const override; + uint16_t load16u(WasmPointer addr) const override; + int32_t load32s(WasmPointer addr) const override; + uint32_t load32u(WasmPointer addr) const override; + int64_t load64s(WasmPointer addr) const override; + uint64_t load64u(WasmPointer addr) const override; + std::array load128(WasmPointer addr) const override; common::Buffer loadN(kagome::runtime::WasmPointer addr, kagome::runtime::WasmSize n) const override; std::string loadStr(kagome::runtime::WasmPointer addr, @@ -57,23 +65,14 @@ namespace kagome::runtime::wasmedge { WasmSpan storeBuffer(gsl::span value) override; - void resize(WasmSize new_size) override { - /** - * We use this condition to avoid - * deallocated_ pointers fixup - */ - if (new_size >= size_) { - size_ = new_size; - memory_->resize(new_size); - } - } + void resize(WasmSize new_size) override {} WasmSize size() const override { return size_; } private: - WasmEdge_ImportObjectContext *memory_; + WasmEdge_MemoryInstanceContext *memory_; WasmSize size_; std::unique_ptr allocator_; @@ -81,4 +80,4 @@ namespace kagome::runtime::wasmedge { }; } // namespace kagome::runtime::wasmedge -#endif // KAGOME_RUNTIME_WASMEDGE_WASM_MEMORY_IMPL_HPP +#endif // KAGOME_RUNTIME_WASMEDGE_MEMORY_IMPL_HPP diff --git a/core/runtime/wasmedge/memory_provider.cpp b/core/runtime/wasmedge/memory_provider.cpp new file mode 100644 index 0000000000..095b703064 --- /dev/null +++ b/core/runtime/wasmedge/memory_provider.cpp @@ -0,0 +1,43 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wasmedge/memory_provider.hpp" + +#include "runtime/common/memory_allocator.hpp" + +#include + +namespace kagome::runtime::wasmedge { + + WasmedgeMemoryProvider::WasmedgeMemoryProvider( + std::shared_ptr memory_factory) + : memory_factory_{std::move(memory_factory)} { + BOOST_ASSERT(memory_factory_); + } + + boost::optional WasmedgeMemoryProvider::getCurrentMemory() + const { + return memory_ == nullptr ? boost::none + : boost::optional{*memory_}; + } + + outcome::result WasmedgeMemoryProvider::resetMemory( + WasmSize heap_base) { + if (store_ctx_) { + WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); + auto memory = WasmEdge_StoreFindMemory(store_ctx_, MemoryName); + if(memory) { + memory_ = memory_factory_->make(memory); + } + } + return outcome::success(); + } + + void WasmedgeMemoryProvider::setExternalInterface( + WasmEdge_StoreContext *storeCtx) { + store_ctx_ = storeCtx; + } + +} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_provider.hpp b/core/runtime/wasmedge/memory_provider.hpp new file mode 100644 index 0000000000..7ccd8b4176 --- /dev/null +++ b/core/runtime/wasmedge/memory_provider.hpp @@ -0,0 +1,37 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CORE_RUNTIME_WASMEDGE_MEMORY_PROVIDER_HPP +#define KAGOME_CORE_RUNTIME_WASMEDGE_MEMORY_PROVIDER_HPP + +#include "runtime/memory_provider.hpp" + +#include "runtime/wasmedge/memory_factory.hpp" + +struct WasmEdge_StoreContext; + +namespace kagome::runtime::wasmedge { + + class WasmedgeMemoryProvider final : public MemoryProvider { + public: + WasmedgeMemoryProvider( + std::shared_ptr memory_factory); + + boost::optional getCurrentMemory() const override; + + [[nodiscard]] outcome::result resetMemory( + WasmSize heap_base) override; + + void setExternalInterface(WasmEdge_StoreContext *storeCxt); + + private: + std::shared_ptr memory_factory_; + std::shared_ptr memory_; + WasmEdge_StoreContext* store_ctx_; + }; + +} // namespace kagome::runtime::wasmedge + +#endif // KAGOME_CORE_RUNTIME_WASMEDGE_MEMORY_PROVIDER_HPP From 8245195649edd6d52896a307d082bd06cc8e736b Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Wed, 8 Sep 2021 16:02:45 +0300 Subject: [PATCH 06/43] add wasmedge module instance --- core/runtime/wasmedge/CMakeLists.txt | 6 ++ .../runtime/wasmedge/module_instance_impl.cpp | 57 +++++++++++++++++++ .../runtime/wasmedge/module_instance_impl.hpp | 38 +++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 core/runtime/wasmedge/module_instance_impl.cpp create mode 100644 core/runtime/wasmedge/module_instance_impl.hpp diff --git a/core/runtime/wasmedge/CMakeLists.txt b/core/runtime/wasmedge/CMakeLists.txt index ea1ae244de..ed4a6b4354 100644 --- a/core/runtime/wasmedge/CMakeLists.txt +++ b/core/runtime/wasmedge/CMakeLists.txt @@ -15,3 +15,9 @@ target_link_libraries(wasmedge_wasm_memory memory_allocator ) kagome_install(wasmedge_wasm_memory) + +add_library(wasmedge_wasm_runtime + module_instance_impl.cpp) +target_link_libraries(wasmedge_wasm_runtime + wasmedge_wasm_memory) +kagome_install(wasmedge_wasm_runtime) diff --git a/core/runtime/wasmedge/module_instance_impl.cpp b/core/runtime/wasmedge/module_instance_impl.cpp new file mode 100644 index 0000000000..749870f5f7 --- /dev/null +++ b/core/runtime/wasmedge/module_instance_impl.cpp @@ -0,0 +1,57 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wasmedge/module_instance_impl.hpp" + +#include + +namespace kagome::runtime::wasmedge { + + ModuleInstanceImpl::ModuleInstanceImpl(WasmEdge_ASTModuleContext *parent, + WasmEdge_ImportObjectContext *rei) + : logger_{log::createLogger("ModuleInstance", "wasmedge")} { + interpreter_ = WasmEdge_InterpreterCreate(nullptr, nullptr); + store_ = WasmEdge_StoreCreate(); + auto ModName = WasmEdge_StringCreateByCString("ext"); + auto Res = WasmEdge_InterpreterRegisterModule( + interpreter_, store_, parent, ModName); + Res = WasmEdge_InterpreterRegisterImport(interpreter_, store_, rei); + WasmEdge_StringDelete(ModName); + } + + outcome::result ModuleInstanceImpl::callExportFunction( + std::string_view name, PtrSize args) const { + auto FuncName = WasmEdge_StringCreateByCString(name.data()); + WasmEdge_Value Params[2] = {WasmEdge_ValueGenI32(args.ptr), + WasmEdge_ValueGenI32(args.size)}; + WasmEdge_Value Returns[1]; + auto Res = WasmEdge_InterpreterInvoke( + interpreter_, store_, FuncName, Params, 2, Returns, 1); + WasmEdge_StringDelete(FuncName); + return PtrSize{WasmEdge_ValueGetI64(Returns[0])}; + } + + outcome::result> ModuleInstanceImpl::getGlobal( + std::string_view name) const { + auto GlobalName = WasmEdge_StringCreateByCString(name.data()); + auto res = WasmEdge_StoreFindGlobal(store_, GlobalName); + auto type = WasmEdge_GlobalInstanceGetValType(res); + auto val = WasmEdge_GlobalInstanceGetValue(res); + WasmEdge_StringDelete(GlobalName); + switch (type) { + case WasmEdge_ValType_I32: + return WasmValue{WasmEdge_ValueGetI32(val)}; + case WasmEdge_ValType_I64: + return WasmValue{WasmEdge_ValueGetI64(val)}; + case WasmEdge_ValType_F32: + return WasmValue{WasmEdge_ValueGetF32(val)}; + case WasmEdge_ValType_F64: + return WasmValue{WasmEdge_ValueGetF64(val)}; + default: + return boost::none; + } + } + +} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_instance_impl.hpp b/core/runtime/wasmedge/module_instance_impl.hpp new file mode 100644 index 0000000000..c608e8b170 --- /dev/null +++ b/core/runtime/wasmedge/module_instance_impl.hpp @@ -0,0 +1,38 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_INSTANCE_IMPL +#define KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_INSTANCE_IMPL + +#include "runtime/module_instance.hpp" + +#include "log/logger.hpp" + +struct WasmEdge_ASTModuleContext; +struct WasmEdge_ImportObjectContext; +struct WasmEdge_InterpreterContext; +struct WasmEdge_StoreContext; + +namespace kagome::runtime::wasmedge { + + class ModuleInstanceImpl final : public ModuleInstance { + public: + ModuleInstanceImpl(WasmEdge_ASTModuleContext *parent, + WasmEdge_ImportObjectContext *rei); + outcome::result callExportFunction(std::string_view name, + PtrSize args) const override; + + outcome::result> getGlobal( + std::string_view name) const override; + + private: + WasmEdge_InterpreterContext *interpreter_; + WasmEdge_StoreContext *store_; + log::Logger logger_; + }; + +} // namespace kagome::runtime::wasmedge + +#endif // KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_INSTANCE_IMPL From 1f6d8e4a8e58bada65d70c9337a1dabf03aa344b Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Thu, 9 Sep 2021 22:37:26 +0300 Subject: [PATCH 07/43] add wasmedge instance environment factory --- core/host_api/impl/host_api_impl.cpp | 81 ---------------- core/host_api/impl/host_api_impl.hpp | 2 - core/runtime/wasmedge/CMakeLists.txt | 2 + .../wasmedge/core_api_factory_impl.cpp | 93 +++++++++++++++++++ .../wasmedge/core_api_factory_impl.hpp | 60 ++++++++++++ .../wasmedge/instance_environment_factory.cpp | 58 ++++++++++++ .../wasmedge/instance_environment_factory.hpp | 56 +++++++++++ core/runtime/wasmedge/memory_factory.cpp | 12 ++- core/runtime/wasmedge/memory_factory.hpp | 4 +- core/runtime/wasmedge/memory_provider.cpp | 21 +++-- core/runtime/wasmedge/memory_provider.hpp | 6 +- .../runtime/wasmedge/module_instance_impl.cpp | 18 +++- .../runtime/wasmedge/module_instance_impl.hpp | 8 +- .../wasmedge/register_host_api.hpp} | 88 +++++++++++++++++- 14 files changed, 404 insertions(+), 105 deletions(-) create mode 100644 core/runtime/wasmedge/core_api_factory_impl.cpp create mode 100644 core/runtime/wasmedge/core_api_factory_impl.hpp create mode 100644 core/runtime/wasmedge/instance_environment_factory.cpp create mode 100644 core/runtime/wasmedge/instance_environment_factory.hpp rename core/{host_api/impl/wasmedge/reg.h => runtime/wasmedge/register_host_api.hpp} (56%) diff --git a/core/host_api/impl/host_api_impl.cpp b/core/host_api/impl/host_api_impl.cpp index 1e01d19457..9974267fb2 100644 --- a/core/host_api/impl/host_api_impl.cpp +++ b/core/host_api/impl/host_api_impl.cpp @@ -63,87 +63,6 @@ namespace kagome::host_api { storage_ext_.reset(); } - void HostApiImpl::reg(WasmEdge_ImportObjectContext *ImpObj) { - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_trie_blake2_256_ordered_root_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_index_set_version_1, ImpObj); - - REGISTER_HOST_API_FUNC(HostApiImpl, ext_logging_log_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_ed25519_generate_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_ed25519_verify_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_finish_batch_verify_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_secp256k1_ecdsa_recover_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApi, - ext_crypto_secp256k1_ecdsa_recover_compressed_version_1, - ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_sr25519_generate_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_sr25519_public_keys_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_sr25519_sign_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_sr25519_verify_version_2, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_start_batch_verify_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_hashing_blake2_128_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_hashing_blake2_256_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_hashing_keccak_256_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_twox_128_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_twox_64_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_free_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_malloc_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_hex_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_utf8_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_misc_runtime_version_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_is_validator_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_local_storage_clear_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_local_storage_compare_and_set_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_local_storage_get_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_local_storage_set_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_network_state_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_random_seed_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_submit_transaction_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_timestamp_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_append_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_changes_root_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_clear_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_clear_prefix_version_2, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_commit_transaction_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_exists_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_get_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_next_key_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_read_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_rollback_transaction_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_root_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_set_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_start_transaction_version_1, ImpObj); - } - runtime::WasmSpan HostApiImpl::ext_storage_read_version_1( runtime::WasmSpan key, runtime::WasmSpan value_out, diff --git a/core/host_api/impl/host_api_impl.hpp b/core/host_api/impl/host_api_impl.hpp index 9ebc954cfd..6949893b53 100644 --- a/core/host_api/impl/host_api_impl.hpp +++ b/core/host_api/impl/host_api_impl.hpp @@ -55,8 +55,6 @@ namespace kagome::host_api { ~HostApiImpl() override = default; - void reg(WasmEdge_ImportObjectContext *ImpObj); - void reset() override; void ext_offchain_index_set_version_1(I64, I64) {} diff --git a/core/runtime/wasmedge/CMakeLists.txt b/core/runtime/wasmedge/CMakeLists.txt index ed4a6b4354..cd6d16c315 100644 --- a/core/runtime/wasmedge/CMakeLists.txt +++ b/core/runtime/wasmedge/CMakeLists.txt @@ -17,6 +17,8 @@ target_link_libraries(wasmedge_wasm_memory kagome_install(wasmedge_wasm_memory) add_library(wasmedge_wasm_runtime + core_api_factory_impl.cpp + instance_environment_factory.cpp module_instance_impl.cpp) target_link_libraries(wasmedge_wasm_runtime wasmedge_wasm_memory) diff --git a/core/runtime/wasmedge/core_api_factory_impl.cpp b/core/runtime/wasmedge/core_api_factory_impl.cpp new file mode 100644 index 0000000000..1854f604b2 --- /dev/null +++ b/core/runtime/wasmedge/core_api_factory_impl.cpp @@ -0,0 +1,93 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wasmedge/core_api_factory_impl.hpp" + +#include "runtime/common/constant_code_provider.hpp" +#include "runtime/common/trie_storage_provider_impl.hpp" +#include "runtime/executor.hpp" +#include "runtime/runtime_api/impl/core.hpp" +#include "runtime/wasmedge/instance_environment_factory.hpp" +#include "runtime/wasmedge/memory_provider.hpp" +#include "runtime/wasmedge/module_instance_impl.hpp" + +#include + +namespace kagome::runtime::wasmedge { + + class OneModuleRepository final : public ModuleRepository { + public: + OneModuleRepository( + WasmEdge_ImportObjectContext *ImpObj, + const std::vector &code, + std::shared_ptr env_factory) + : env_factory_{std::move(env_factory)}, code_{code}, imp_obj_{ImpObj} { + BOOST_ASSERT(env_factory_); + } + + outcome::result> getInstanceAt( + std::shared_ptr, + const primitives::BlockInfo &b) override { + if (instance_ == nullptr) { + WasmEdge_LoaderContext *LoaderCtx = WasmEdge_LoaderCreate(nullptr); + WasmEdge_ASTModuleContext *ASTCtx = nullptr; + auto Res = WasmEdge_LoaderParseFromBuffer( + LoaderCtx, &ASTCtx, code_.data(), code_.size()); + auto env = env_factory_->make(); + instance_ = std::dynamic_pointer_cast( + std::make_shared( + std::move(env.env), ASTCtx, imp_obj_)); + } + return instance_; + } + + private: + std::shared_ptr instance_; + std::shared_ptr env_factory_; + const std::vector &code_; + WasmEdge_ImportObjectContext *imp_obj_; + }; + + class OneCodeProvider final : public RuntimeCodeProvider { + public: + explicit OneCodeProvider(gsl::span code) : code_{code} {} + + virtual outcome::result> getCodeAt( + const storage::trie::RootHash &) const { + return code_; + } + + private: + gsl::span code_; + }; + + CoreApiFactoryImpl::CoreApiFactoryImpl( + WasmEdge_ImportObjectContext *ImpObj, + std::shared_ptr instance_env_factory, + std::shared_ptr header_repo, + std::shared_ptr changes_tracker) + : instance_env_factory_{std::move(instance_env_factory)}, + header_repo_{std::move(header_repo)}, + changes_tracker_{std::move(changes_tracker)}, + imp_obj_{ImpObj} { + BOOST_ASSERT(instance_env_factory_ != nullptr); + BOOST_ASSERT(header_repo_ != nullptr); + BOOST_ASSERT(changes_tracker_ != nullptr); + } + + std::unique_ptr CoreApiFactoryImpl::make( + std::shared_ptr hasher, + const std::vector &runtime_code) const { + auto env_factory = std::make_shared( + std::make_shared(runtime_code), + std::make_shared( + imp_obj_, runtime_code, instance_env_factory_), + header_repo_); + auto executor = std::make_unique(header_repo_, env_factory); + return std::make_unique( + std::move(executor), changes_tracker_, header_repo_); + } + +} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/core_api_factory_impl.hpp b/core/runtime/wasmedge/core_api_factory_impl.hpp new file mode 100644 index 0000000000..ee052c26df --- /dev/null +++ b/core/runtime/wasmedge/core_api_factory_impl.hpp @@ -0,0 +1,60 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CORE_RUNTIME_WASMEDGE_CORE_API_FACTORY_IMPL_HPP +#define KAGOME_CORE_RUNTIME_WASMEDGE_CORE_API_FACTORY_IMPL_HPP + +#include "runtime/core_api_factory_impl.hpp" +#include "runtime/runtime_api/core.hpp" + +namespace kagome::storage::changes_trie { + class ChangesTracker; +} + +namespace kagome::storage::trie { + class TrieStorage; +} + +namespace kagome::blockchain { + class BlockHeaderRepository; +} + +namespace kagome::runtime { + class TrieStorageProvider; + class Memory; + class RuntimeEnvironmentFactory; +} // namespace kagome::runtime + +struct WasmEdge_ImportObjectContext; + +namespace kagome::runtime::wasmedge { + + class InstanceEnvironmentFactory; + class WasmedgeMemoryFactory; + + class CoreApiFactoryImpl final + : public runtime::CoreApiFactory, + public std::enable_shared_from_this { + public: + CoreApiFactoryImpl( + WasmEdge_ImportObjectContext *ImpObj, + std::shared_ptr instance_env_factory, + std::shared_ptr header_repo, + std::shared_ptr changes_tracker); + + std::unique_ptr make( + std::shared_ptr hasher, + const std::vector &runtime_code) const override; + + private: + std::shared_ptr instance_env_factory_; + std::shared_ptr header_repo_; + std::shared_ptr changes_tracker_; + WasmEdge_ImportObjectContext *imp_obj_; + }; + +} // namespace kagome::runtime::wasmedge + +#endif // KAGOME_CORE_RUNTIME_WASMEDGE_CORE_API_FACTORY_IMPL_HPP diff --git a/core/runtime/wasmedge/instance_environment_factory.cpp b/core/runtime/wasmedge/instance_environment_factory.cpp new file mode 100644 index 0000000000..6dad942044 --- /dev/null +++ b/core/runtime/wasmedge/instance_environment_factory.cpp @@ -0,0 +1,58 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wasmedge/instance_environment_factory.hpp" + +#include "host_api/host_api_factory.hpp" +#include "host_api/impl/host_api_impl.hpp" +#include "runtime/common/trie_storage_provider_impl.hpp" +#include "runtime/wasmedge/core_api_factory_impl.hpp" +#include "runtime/wasmedge/memory_provider.hpp" +#include "runtime/wasmedge/register_host_api.hpp" + +#include + +namespace kagome::runtime::wasmedge { + + InstanceEnvironmentFactory::InstanceEnvironmentFactory( + std::shared_ptr storage, + std::shared_ptr host_api_factory, + std::shared_ptr block_header_repo, + std::shared_ptr changes_tracker) + : storage_{std::move(storage)}, + host_api_factory_{std::move(host_api_factory)}, + block_header_repo_{std::move(block_header_repo)}, + changes_tracker_{std::move(changes_tracker)} { + BOOST_ASSERT(storage_); + BOOST_ASSERT(host_api_factory_); + BOOST_ASSERT(block_header_repo_); + BOOST_ASSERT(changes_tracker_); + } + + WasmedgeInstanceEnvironment InstanceEnvironmentFactory::make() const { + WasmEdge_VMContext *VMCxt = WasmEdge_VMCreate(NULL, NULL); + WasmEdge_String ExportName = WasmEdge_StringCreateByCString("ext"); + WasmEdge_ImportObjectContext *ImpObj = + WasmEdge_ImportObjectCreate(ExportName, NULL); + WasmEdge_StringDelete(ExportName); + register_host_api(ImpObj); + auto memory_factory = std::make_shared(); + auto new_memory_provider = + std::make_shared(memory_factory); + new_memory_provider->setExternalInterface(ImpObj); + auto new_storage_provider = + std::make_shared(storage_); + auto core_factory = std::make_shared( + ImpObj, shared_from_this(), block_header_repo_, changes_tracker_); + auto host_api = std::shared_ptr(host_api_factory_->make( + core_factory, new_memory_provider, new_storage_provider)); + return WasmedgeInstanceEnvironment{ + InstanceEnvironment{std::move(new_memory_provider), + std::move(new_storage_provider), + std::move(host_api), + [](auto &) {}}, + VMCxt}; + } +} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/instance_environment_factory.hpp b/core/runtime/wasmedge/instance_environment_factory.hpp new file mode 100644 index 0000000000..93438e648c --- /dev/null +++ b/core/runtime/wasmedge/instance_environment_factory.hpp @@ -0,0 +1,56 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CORE_RUNTIME_WASMEDGE_INSTANCE_ENVIRONMENT_FACTORY_HPP +#define KAGOME_CORE_RUNTIME_WASMEDGE_INSTANCE_ENVIRONMENT_FACTORY_HPP + +#include "runtime/instance_environment.hpp" + +namespace kagome::storage::trie { + class TrieStorage; +} + +namespace kagome::storage::changes_trie { + class ChangesTracker; +} + +namespace kagome::host_api { + class HostApiFactory; +} + +namespace kagome::blockchain { + class BlockHeaderRepository; +} + +struct WasmEdge_VMContext; + +namespace kagome::runtime::wasmedge { + + struct WasmedgeInstanceEnvironment { + InstanceEnvironment env; + WasmEdge_VMContext* vm; + }; + + class InstanceEnvironmentFactory final + : public std::enable_shared_from_this { + public: + InstanceEnvironmentFactory( + std::shared_ptr storage, + std::shared_ptr host_api_factory, + std::shared_ptr block_header_repo, + std::shared_ptr changes_tracker); + + [[nodiscard]] WasmedgeInstanceEnvironment make() const; + + private: + std::shared_ptr storage_; + std::shared_ptr host_api_factory_; + std::shared_ptr block_header_repo_; + std::shared_ptr changes_tracker_; + }; + +} // namespace kagome::runtime::wasmedge + +#endif // KAGOME_CORE_RUNTIME_WASMEDGE_INSTANCE_ENVIRONMENT_FACTORY_HPP diff --git a/core/runtime/wasmedge/memory_factory.cpp b/core/runtime/wasmedge/memory_factory.cpp index 15fb8e649e..4c6f34a144 100644 --- a/core/runtime/wasmedge/memory_factory.cpp +++ b/core/runtime/wasmedge/memory_factory.cpp @@ -7,11 +7,19 @@ #include "runtime/common/memory_allocator.hpp" +#include + namespace kagome::runtime::wasmedge { std::unique_ptr WasmedgeMemoryFactory::make( - WasmEdge_MemoryInstanceContext *memory) const { - return std::make_unique(memory); + WasmEdge_ImportObjectContext *obj) const { + WasmEdge_Limit MemoryLimit = {.HasMax = false, .Min = 40, .Max = 40}; + WasmEdge_MemoryInstanceContext *HostMemory = + WasmEdge_MemoryInstanceCreate(MemoryLimit); + WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); + WasmEdge_ImportObjectAddMemory(obj, MemoryName, HostMemory); + WasmEdge_StringDelete(MemoryName); + return std::make_unique(HostMemory); } } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_factory.hpp b/core/runtime/wasmedge/memory_factory.hpp index cb341f03a7..6064d3f762 100644 --- a/core/runtime/wasmedge/memory_factory.hpp +++ b/core/runtime/wasmedge/memory_factory.hpp @@ -8,6 +8,8 @@ #include "runtime/wasmedge/memory_impl.hpp" +struct WasmEdge_ImportObjectContext; + namespace kagome::runtime::wasmedge { class WasmedgeMemoryFactory { @@ -15,7 +17,7 @@ namespace kagome::runtime::wasmedge { virtual ~WasmedgeMemoryFactory() = default; std::unique_ptr make( - WasmEdge_MemoryInstanceContext *memory) const; + WasmEdge_ImportObjectContext *obj) const; }; } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_provider.cpp b/core/runtime/wasmedge/memory_provider.cpp index 095b703064..e21886e46d 100644 --- a/core/runtime/wasmedge/memory_provider.cpp +++ b/core/runtime/wasmedge/memory_provider.cpp @@ -25,19 +25,24 @@ namespace kagome::runtime::wasmedge { outcome::result WasmedgeMemoryProvider::resetMemory( WasmSize heap_base) { - if (store_ctx_) { - WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); - auto memory = WasmEdge_StoreFindMemory(store_ctx_, MemoryName); - if(memory) { - memory_ = memory_factory_->make(memory); - } + /* + * if (store_ctx_) { + * WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); + * auto memory = WasmEdge_StoreFindMemory(store_ctx_, MemoryName); + * if(memory) { + * memory_ = memory_factory_->make(memory); + * } + * } + */ + if(imp_obj_) { + memory_ = memory_factory_->make(imp_obj_); } return outcome::success(); } void WasmedgeMemoryProvider::setExternalInterface( - WasmEdge_StoreContext *storeCtx) { - store_ctx_ = storeCtx; + WasmEdge_ImportObjectContext *impObj) { + imp_obj_ = impObj; } } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_provider.hpp b/core/runtime/wasmedge/memory_provider.hpp index 7ccd8b4176..dafb4cbecd 100644 --- a/core/runtime/wasmedge/memory_provider.hpp +++ b/core/runtime/wasmedge/memory_provider.hpp @@ -10,7 +10,7 @@ #include "runtime/wasmedge/memory_factory.hpp" -struct WasmEdge_StoreContext; +struct WasmEdge_ImportObjectContext; namespace kagome::runtime::wasmedge { @@ -24,12 +24,12 @@ namespace kagome::runtime::wasmedge { [[nodiscard]] outcome::result resetMemory( WasmSize heap_base) override; - void setExternalInterface(WasmEdge_StoreContext *storeCxt); + void setExternalInterface(WasmEdge_ImportObjectContext *impObj); private: std::shared_ptr memory_factory_; std::shared_ptr memory_; - WasmEdge_StoreContext* store_ctx_; + WasmEdge_ImportObjectContext *imp_obj_; }; } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_instance_impl.cpp b/core/runtime/wasmedge/module_instance_impl.cpp index 749870f5f7..bf1a08a02d 100644 --- a/core/runtime/wasmedge/module_instance_impl.cpp +++ b/core/runtime/wasmedge/module_instance_impl.cpp @@ -5,13 +5,18 @@ #include "runtime/wasmedge/module_instance_impl.hpp" +#include "host_api/host_api.hpp" + #include namespace kagome::runtime::wasmedge { - ModuleInstanceImpl::ModuleInstanceImpl(WasmEdge_ASTModuleContext *parent, + ModuleInstanceImpl::ModuleInstanceImpl(InstanceEnvironment &&env, + WasmEdge_ASTModuleContext *parent, WasmEdge_ImportObjectContext *rei) - : logger_{log::createLogger("ModuleInstance", "wasmedge")} { + + : env_{std::move(env)}, + logger_{log::createLogger("ModuleInstance", "wasmedge")} { interpreter_ = WasmEdge_InterpreterCreate(nullptr, nullptr); store_ = WasmEdge_StoreCreate(); auto ModName = WasmEdge_StringCreateByCString("ext"); @@ -54,4 +59,13 @@ namespace kagome::runtime::wasmedge { } } + InstanceEnvironment const &ModuleInstanceImpl::getEnvironment() const { + return env_; + } + + outcome::result ModuleInstanceImpl::resetEnvironment() { + env_.host_api->reset(); + return outcome::success(); + } + } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_instance_impl.hpp b/core/runtime/wasmedge/module_instance_impl.hpp index c608e8b170..09cdf38d22 100644 --- a/core/runtime/wasmedge/module_instance_impl.hpp +++ b/core/runtime/wasmedge/module_instance_impl.hpp @@ -19,7 +19,8 @@ namespace kagome::runtime::wasmedge { class ModuleInstanceImpl final : public ModuleInstance { public: - ModuleInstanceImpl(WasmEdge_ASTModuleContext *parent, + ModuleInstanceImpl(InstanceEnvironment &&env, + WasmEdge_ASTModuleContext *parent, WasmEdge_ImportObjectContext *rei); outcome::result callExportFunction(std::string_view name, PtrSize args) const override; @@ -27,7 +28,12 @@ namespace kagome::runtime::wasmedge { outcome::result> getGlobal( std::string_view name) const override; + InstanceEnvironment const &getEnvironment() const override; + + outcome::result resetEnvironment() override; + private: + InstanceEnvironment env_; WasmEdge_InterpreterContext *interpreter_; WasmEdge_StoreContext *store_; log::Logger logger_; diff --git a/core/host_api/impl/wasmedge/reg.h b/core/runtime/wasmedge/register_host_api.hpp similarity index 56% rename from core/host_api/impl/wasmedge/reg.h rename to core/runtime/wasmedge/register_host_api.hpp index 154e1ba942..dbe0a952e8 100644 --- a/core/host_api/impl/wasmedge/reg.h +++ b/core/runtime/wasmedge/register_host_api.hpp @@ -62,7 +62,7 @@ struct MakeSigned { using type = typename std:: conditional_t, typename std::make_signed_t, T>; }; -template<> +template <> struct MakeSigned { using type = void; }; @@ -169,11 +169,11 @@ struct HostApiFunc { static WasmEdge_FunctionTypeContext *type() { enum WasmEdge_ValType ParamList[sizeof...(Args)] = { - ::type::type>()...}; + ::type::type>()...}; size_t r = 1; enum WasmEdge_ValType ReturnListNonVoid[1] = { - ::type::type>()}; + ::type::type>()}; enum WasmEdge_ValType *ReturnList = ReturnListNonVoid; if constexpr (std::is_same_v) { @@ -182,11 +182,10 @@ struct HostApiFunc { } return WasmEdge_FunctionTypeCreate( - ParamList, sizeof...(Args), ReturnList, r); + ParamList, sizeof...(Args), ReturnList, r); } }; - template void registerHostApiFunc(const std::string &name, WasmEdge_ImportObjectContext *ImpObj) { @@ -202,3 +201,82 @@ void registerHostApiFunc(const std::string &name, #define REGISTER_HOST_API_FUNC(class, name, obj) \ registerHostApiFunc<&class ::name>(#name, obj) + +namespace { + using kagome::host_api::HostApi; + using kagome::host_api::HostApiImpl; +} + +inline void register_host_api(WasmEdge_ImportObjectContext *ImpObj) { + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_trie_blake2_256_ordered_root_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_offchain_index_set_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_logging_log_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_ed25519_generate_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_ed25519_verify_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_finish_batch_verify_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_secp256k1_ecdsa_recover_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApi, ext_crypto_secp256k1_ecdsa_recover_compressed_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_sr25519_generate_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_sr25519_public_keys_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_sr25519_sign_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_sr25519_verify_version_2, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_start_batch_verify_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_blake2_128_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_blake2_256_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_keccak_256_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_twox_128_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_twox_64_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_free_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_malloc_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_hex_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_utf8_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_misc_runtime_version_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_is_validator_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_local_storage_clear_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, + ext_offchain_local_storage_compare_and_set_version_1, + ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_local_storage_get_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_local_storage_set_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_network_state_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_random_seed_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_offchain_submit_transaction_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_offchain_timestamp_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_append_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_changes_root_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_clear_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_clear_prefix_version_2, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_commit_transaction_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_exists_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_get_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_next_key_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_read_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_rollback_transaction_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_root_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_set_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_start_transaction_version_1, ImpObj); +} From 0c117ecc1f6fe9605e3de92a186cb30f4361a69c Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Fri, 10 Sep 2021 08:32:08 +0300 Subject: [PATCH 08/43] add wasmedge option --- core/application/app_configuration.hpp | 6 +++- .../impl/app_configuration_impl.cpp | 5 ++- core/injector/CMakeLists.txt | 2 ++ core/injector/application_injector.cpp | 18 ++++++++--- core/runtime/wasmedge/CMakeLists.txt | 1 - .../wasmedge/instance_environment_factory.cpp | 24 +++++++------- .../wasmedge/instance_environment_factory.hpp | 6 +++- core/runtime/wasmedge/memory_factory.cpp | 25 --------------- core/runtime/wasmedge/memory_factory.hpp | 25 --------------- core/runtime/wasmedge/memory_provider.cpp | 32 ++++++------------- core/runtime/wasmedge/memory_provider.hpp | 11 +++---- 11 files changed, 56 insertions(+), 99 deletions(-) delete mode 100644 core/runtime/wasmedge/memory_factory.cpp delete mode 100644 core/runtime/wasmedge/memory_factory.hpp diff --git a/core/application/app_configuration.hpp b/core/application/app_configuration.hpp index 95d306ad1e..24c7c5770f 100644 --- a/core/application/app_configuration.hpp +++ b/core/application/app_configuration.hpp @@ -149,7 +149,11 @@ namespace kagome::application { */ virtual const std::string &nodeVersion() const = 0; - enum class RuntimeExecutionMethod { Compile, Interpret }; + enum class RuntimeExecutionMethod { + Compile, + Interpret, + WasmEdge + }; /** * @return enum constant of the chosen runtime backend */ diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index 97a4b7c542..0dd4ac8c64 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -81,7 +81,10 @@ namespace { if (str == "Compiled") { return REM::Compile; } - return std::nullopt; + if (str == "WasmEdge") { + return REM::WasmEdge; + } + return std::nullopt_t; } std::optional diff --git a/core/injector/CMakeLists.txt b/core/injector/CMakeLists.txt index 0341aa118a..a48fcb63ea 100644 --- a/core/injector/CMakeLists.txt +++ b/core/injector/CMakeLists.txt @@ -117,4 +117,6 @@ target_link_libraries(application_injector offchain_worker_api offchain_persistent_storage offchain_local_storage + wasmedge_wasm_memory + wasmedge_wasm_runtime ) diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 7bac2fd33f..553df1c906 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -113,6 +113,8 @@ #include "runtime/runtime_api/impl/parachain_host.hpp" #include "runtime/runtime_api/impl/tagged_transaction_queue.hpp" #include "runtime/runtime_api/impl/transaction_payment_api.hpp" +#include "runtime/wasmedge/core_api_factory_impl.hpp" +#include "runtime/wasmedge/memory_provider.hpp" #include "runtime/wavm/compartment_wrapper.hpp" #include "runtime/wavm/core_api_factory_impl.hpp" #include "runtime/wavm/instance_environment_factory.hpp" @@ -727,7 +729,7 @@ namespace { template auto makeWavmInjector( application::AppConfiguration::RuntimeExecutionMethod method, - Ts &&...args) { + Ts &&... args) { return di::make_injector( di::bind.template to( [](const auto &injector) { @@ -768,7 +770,7 @@ namespace { template auto makeBinaryenInjector( application::AppConfiguration::RuntimeExecutionMethod method, - Ts &&...args) { + Ts &&... args) { return di::make_injector( di::bind.template to( [](const auto &injector) { @@ -789,6 +791,7 @@ namespace { template auto choose_runtime_implementation( @@ -804,6 +807,9 @@ namespace { case RuntimeExecutionMethod::Compile: return std::static_pointer_cast( injector.template create>()); + case RuntimeExecutionMethod::WasmEdge: + return std::static_pointer_cast( + injector.template create>()); } throw std::runtime_error("Unknown runtime execution method"); }(); @@ -839,7 +845,7 @@ namespace { template auto makeRuntimeInjector( application::AppConfiguration::RuntimeExecutionMethod method, - Ts &&...args) { + Ts &&... args) { return di::make_injector( di::bind.template to(), di::bind.template to( @@ -858,6 +864,7 @@ namespace { return choose_runtime_implementation< runtime::CoreApiFactory, runtime::binaryen::CoreApiFactoryImpl, + runtime::wasmedge::CoreApiFactoryImpl, runtime::wavm::CoreApiFactoryImpl>(injector, method); }), di::bind.template to( @@ -865,6 +872,7 @@ namespace { return choose_runtime_implementation< runtime::ModuleFactory, runtime::binaryen::ModuleFactoryImpl, + runtime::binaryen::ModuleFactoryImpl, runtime::wavm::ModuleFactoryImpl>(injector, method); }), di::bind.template to([](const auto &injector) { @@ -900,7 +908,7 @@ namespace { template auto makeApplicationInjector(const application::AppConfiguration &config, - Ts &&...args) { + Ts &&... args) { // default values for configurations api::RpcThreadPool::Configuration rpc_thread_pool_config{}; api::HttpSession::Configuration http_config{}; @@ -1338,7 +1346,7 @@ namespace { template auto makeKagomeNodeInjector(const application::AppConfiguration &app_config, - Ts &&...args) { + Ts &&... args) { using namespace boost; // NOLINT; return di::make_injector( diff --git a/core/runtime/wasmedge/CMakeLists.txt b/core/runtime/wasmedge/CMakeLists.txt index cd6d16c315..457b3739a1 100644 --- a/core/runtime/wasmedge/CMakeLists.txt +++ b/core/runtime/wasmedge/CMakeLists.txt @@ -4,7 +4,6 @@ # add_library(wasmedge_wasm_memory - memory_factory.cpp memory_impl.cpp memory_provider.cpp ) diff --git a/core/runtime/wasmedge/instance_environment_factory.cpp b/core/runtime/wasmedge/instance_environment_factory.cpp index 6dad942044..98de8500ad 100644 --- a/core/runtime/wasmedge/instance_environment_factory.cpp +++ b/core/runtime/wasmedge/instance_environment_factory.cpp @@ -29,30 +29,32 @@ namespace kagome::runtime::wasmedge { BOOST_ASSERT(host_api_factory_); BOOST_ASSERT(block_header_repo_); BOOST_ASSERT(changes_tracker_); - } - WasmedgeInstanceEnvironment InstanceEnvironmentFactory::make() const { - WasmEdge_VMContext *VMCxt = WasmEdge_VMCreate(NULL, NULL); + vm_ = WasmEdge_VMCreate(NULL, NULL); WasmEdge_String ExportName = WasmEdge_StringCreateByCString("ext"); WasmEdge_ImportObjectContext *ImpObj = WasmEdge_ImportObjectCreate(ExportName, NULL); WasmEdge_StringDelete(ExportName); register_host_api(ImpObj); - auto memory_factory = std::make_shared(); - auto new_memory_provider = - std::make_shared(memory_factory); - new_memory_provider->setExternalInterface(ImpObj); + memory_provider_ = std::make_shared(); + memory_provider_->setExternalInterface(ImpObj); + WasmEdge_VMRegisterModuleFromImport(vm_, ImpObj); + } + + WasmedgeInstanceEnvironment InstanceEnvironmentFactory::make() const { + auto imp_obj = WasmEdge_VMGetImportModuleContext(vm_, WasmEdge_HostRegistration_Wasi); + memory_provider_->resetMemory(0); auto new_storage_provider = std::make_shared(storage_); auto core_factory = std::make_shared( - ImpObj, shared_from_this(), block_header_repo_, changes_tracker_); + imp_obj, shared_from_this(), block_header_repo_, changes_tracker_); auto host_api = std::shared_ptr(host_api_factory_->make( - core_factory, new_memory_provider, new_storage_provider)); + core_factory, memory_provider_, new_storage_provider)); return WasmedgeInstanceEnvironment{ - InstanceEnvironment{std::move(new_memory_provider), + InstanceEnvironment{memory_provider_, std::move(new_storage_provider), std::move(host_api), [](auto &) {}}, - VMCxt}; + vm_}; } } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/instance_environment_factory.hpp b/core/runtime/wasmedge/instance_environment_factory.hpp index 93438e648c..a57e25c3b4 100644 --- a/core/runtime/wasmedge/instance_environment_factory.hpp +++ b/core/runtime/wasmedge/instance_environment_factory.hpp @@ -28,9 +28,11 @@ struct WasmEdge_VMContext; namespace kagome::runtime::wasmedge { + class WasmedgeMemoryProvider; + struct WasmedgeInstanceEnvironment { InstanceEnvironment env; - WasmEdge_VMContext* vm; + WasmEdge_VMContext *vm; }; class InstanceEnvironmentFactory final @@ -49,6 +51,8 @@ namespace kagome::runtime::wasmedge { std::shared_ptr host_api_factory_; std::shared_ptr block_header_repo_; std::shared_ptr changes_tracker_; + std::shared_ptr memory_provider_; + WasmEdge_VMContext *vm_; }; } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_factory.cpp b/core/runtime/wasmedge/memory_factory.cpp deleted file mode 100644 index 4c6f34a144..0000000000 --- a/core/runtime/wasmedge/memory_factory.cpp +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "runtime/wasmedge/memory_factory.hpp" - -#include "runtime/common/memory_allocator.hpp" - -#include - -namespace kagome::runtime::wasmedge { - - std::unique_ptr WasmedgeMemoryFactory::make( - WasmEdge_ImportObjectContext *obj) const { - WasmEdge_Limit MemoryLimit = {.HasMax = false, .Min = 40, .Max = 40}; - WasmEdge_MemoryInstanceContext *HostMemory = - WasmEdge_MemoryInstanceCreate(MemoryLimit); - WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); - WasmEdge_ImportObjectAddMemory(obj, MemoryName, HostMemory); - WasmEdge_StringDelete(MemoryName); - return std::make_unique(HostMemory); - } - -} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_factory.hpp b/core/runtime/wasmedge/memory_factory.hpp deleted file mode 100644 index 6064d3f762..0000000000 --- a/core/runtime/wasmedge/memory_factory.hpp +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef KAGOME_WASMEDGE_MEMORY_FACTORY_HPP -#define KAGOME_WASMEDGE_MEMORY_FACTORY_HPP - -#include "runtime/wasmedge/memory_impl.hpp" - -struct WasmEdge_ImportObjectContext; - -namespace kagome::runtime::wasmedge { - - class WasmedgeMemoryFactory { - public: - virtual ~WasmedgeMemoryFactory() = default; - - std::unique_ptr make( - WasmEdge_ImportObjectContext *obj) const; - }; - -} // namespace kagome::runtime::wasmedge - -#endif // KAGOME_WASMEDGE_MEMORY_FACTORY_HPP diff --git a/core/runtime/wasmedge/memory_provider.cpp b/core/runtime/wasmedge/memory_provider.cpp index e21886e46d..15200546fc 100644 --- a/core/runtime/wasmedge/memory_provider.cpp +++ b/core/runtime/wasmedge/memory_provider.cpp @@ -6,43 +6,31 @@ #include "runtime/wasmedge/memory_provider.hpp" #include "runtime/common/memory_allocator.hpp" +#include "runtime/wasmedge/memory_impl.hpp" #include namespace kagome::runtime::wasmedge { - WasmedgeMemoryProvider::WasmedgeMemoryProvider( - std::shared_ptr memory_factory) - : memory_factory_{std::move(memory_factory)} { - BOOST_ASSERT(memory_factory_); - } - boost::optional WasmedgeMemoryProvider::getCurrentMemory() const { return memory_ == nullptr ? boost::none : boost::optional{*memory_}; } - outcome::result WasmedgeMemoryProvider::resetMemory( - WasmSize heap_base) { - /* - * if (store_ctx_) { - * WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); - * auto memory = WasmEdge_StoreFindMemory(store_ctx_, MemoryName); - * if(memory) { - * memory_ = memory_factory_->make(memory); - * } - * } - */ - if(imp_obj_) { - memory_ = memory_factory_->make(imp_obj_); - } + outcome::result WasmedgeMemoryProvider::resetMemory(WasmSize) { + memory_ = std::make_unique(mem_ctx_); return outcome::success(); } void WasmedgeMemoryProvider::setExternalInterface( - WasmEdge_ImportObjectContext *impObj) { - imp_obj_ = impObj; + WasmEdge_ImportObjectContext *imp_obj) { + WasmEdge_Limit MemoryLimit = {.HasMax = false, .Min = 40, .Max = 40}; + mem_ctx_ = WasmEdge_MemoryInstanceCreate(MemoryLimit); + WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); + WasmEdge_ImportObjectAddMemory(imp_obj, MemoryName, mem_ctx_); + WasmEdge_StringDelete(MemoryName); + resetMemory(0); } } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/memory_provider.hpp b/core/runtime/wasmedge/memory_provider.hpp index dafb4cbecd..7a0ff38836 100644 --- a/core/runtime/wasmedge/memory_provider.hpp +++ b/core/runtime/wasmedge/memory_provider.hpp @@ -8,17 +8,15 @@ #include "runtime/memory_provider.hpp" -#include "runtime/wasmedge/memory_factory.hpp" - struct WasmEdge_ImportObjectContext; +struct WasmEdge_MemoryInstanceContext; namespace kagome::runtime::wasmedge { + class MemoryImpl; + class WasmedgeMemoryProvider final : public MemoryProvider { public: - WasmedgeMemoryProvider( - std::shared_ptr memory_factory); - boost::optional getCurrentMemory() const override; [[nodiscard]] outcome::result resetMemory( @@ -27,9 +25,8 @@ namespace kagome::runtime::wasmedge { void setExternalInterface(WasmEdge_ImportObjectContext *impObj); private: - std::shared_ptr memory_factory_; std::shared_ptr memory_; - WasmEdge_ImportObjectContext *imp_obj_; + WasmEdge_MemoryInstanceContext* mem_ctx_; }; } // namespace kagome::runtime::wasmedge From ec6be5e3bd92484cf1035dcd99b8932ff238559c Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Fri, 10 Sep 2021 09:30:15 +0300 Subject: [PATCH 09/43] add wasmedge module --- .../impl/app_configuration_impl.cpp | 2 +- core/injector/application_injector.cpp | 4 +- core/runtime/wasmedge/CMakeLists.txt | 2 + .../wasmedge/core_api_factory_impl.cpp | 26 +++----- .../wasmedge/core_api_factory_impl.hpp | 4 -- .../wasmedge/instance_environment_factory.cpp | 3 +- core/runtime/wasmedge/module_factory_impl.cpp | 36 +++++++++++ core/runtime/wasmedge/module_factory_impl.hpp | 51 +++++++++++++++ core/runtime/wasmedge/module_impl.cpp | 50 +++++++++++++++ core/runtime/wasmedge/module_impl.hpp | 63 +++++++++++++++++++ .../runtime/wasmedge/module_instance_impl.cpp | 21 +++---- .../runtime/wasmedge/module_instance_impl.hpp | 12 ++-- 12 files changed, 231 insertions(+), 43 deletions(-) create mode 100644 core/runtime/wasmedge/module_factory_impl.cpp create mode 100644 core/runtime/wasmedge/module_factory_impl.hpp create mode 100644 core/runtime/wasmedge/module_impl.cpp create mode 100644 core/runtime/wasmedge/module_impl.hpp diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index 0dd4ac8c64..2bd761191f 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -501,7 +501,7 @@ namespace kagome::application { ("dev", "if node run in development mode") ("dev-with-wipe", "if needed to wipe base path (only for dev mode)") ("wasm-execution", po::value()->default_value("Interpreted"), - "choose the desired wasm execution method (Compiled, Interpreted)") + "choose the desired wasm execution method (Compiled, Interpreted, WasmEdge)") ; // clang-format on diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 553df1c906..22210ac11c 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -114,7 +114,9 @@ #include "runtime/runtime_api/impl/tagged_transaction_queue.hpp" #include "runtime/runtime_api/impl/transaction_payment_api.hpp" #include "runtime/wasmedge/core_api_factory_impl.hpp" +#include "runtime/wasmedge/instance_environment_factory.hpp" #include "runtime/wasmedge/memory_provider.hpp" +#include "runtime/wasmedge/module_factory_impl.hpp" #include "runtime/wavm/compartment_wrapper.hpp" #include "runtime/wavm/core_api_factory_impl.hpp" #include "runtime/wavm/instance_environment_factory.hpp" @@ -872,7 +874,7 @@ namespace { return choose_runtime_implementation< runtime::ModuleFactory, runtime::binaryen::ModuleFactoryImpl, - runtime::binaryen::ModuleFactoryImpl, + runtime::wasmedge::ModuleFactoryImpl, runtime::wavm::ModuleFactoryImpl>(injector, method); }), di::bind.template to([](const auto &injector) { diff --git a/core/runtime/wasmedge/CMakeLists.txt b/core/runtime/wasmedge/CMakeLists.txt index 457b3739a1..c9cf825a2c 100644 --- a/core/runtime/wasmedge/CMakeLists.txt +++ b/core/runtime/wasmedge/CMakeLists.txt @@ -18,6 +18,8 @@ kagome_install(wasmedge_wasm_memory) add_library(wasmedge_wasm_runtime core_api_factory_impl.cpp instance_environment_factory.cpp + module_factory_impl.cpp + module_impl.cpp module_instance_impl.cpp) target_link_libraries(wasmedge_wasm_runtime wasmedge_wasm_memory) diff --git a/core/runtime/wasmedge/core_api_factory_impl.cpp b/core/runtime/wasmedge/core_api_factory_impl.cpp index 1854f604b2..b31de750b5 100644 --- a/core/runtime/wasmedge/core_api_factory_impl.cpp +++ b/core/runtime/wasmedge/core_api_factory_impl.cpp @@ -11,19 +11,17 @@ #include "runtime/runtime_api/impl/core.hpp" #include "runtime/wasmedge/instance_environment_factory.hpp" #include "runtime/wasmedge/memory_provider.hpp" +#include "runtime/wasmedge/module_impl.hpp" #include "runtime/wasmedge/module_instance_impl.hpp" -#include - namespace kagome::runtime::wasmedge { class OneModuleRepository final : public ModuleRepository { public: OneModuleRepository( - WasmEdge_ImportObjectContext *ImpObj, const std::vector &code, std::shared_ptr env_factory) - : env_factory_{std::move(env_factory)}, code_{code}, imp_obj_{ImpObj} { + : env_factory_{std::move(env_factory)}, code_{code} { BOOST_ASSERT(env_factory_); } @@ -31,14 +29,9 @@ namespace kagome::runtime::wasmedge { std::shared_ptr, const primitives::BlockInfo &b) override { if (instance_ == nullptr) { - WasmEdge_LoaderContext *LoaderCtx = WasmEdge_LoaderCreate(nullptr); - WasmEdge_ASTModuleContext *ASTCtx = nullptr; - auto Res = WasmEdge_LoaderParseFromBuffer( - LoaderCtx, &ASTCtx, code_.data(), code_.size()); - auto env = env_factory_->make(); - instance_ = std::dynamic_pointer_cast( - std::make_shared( - std::move(env.env), ASTCtx, imp_obj_)); + OUTCOME_TRY(module, ModuleImpl::createFromCode(code_, env_factory_)); + OUTCOME_TRY(inst, module->instantiate()); + instance_ = std::move(inst); } return instance_; } @@ -47,7 +40,6 @@ namespace kagome::runtime::wasmedge { std::shared_ptr instance_; std::shared_ptr env_factory_; const std::vector &code_; - WasmEdge_ImportObjectContext *imp_obj_; }; class OneCodeProvider final : public RuntimeCodeProvider { @@ -64,14 +56,12 @@ namespace kagome::runtime::wasmedge { }; CoreApiFactoryImpl::CoreApiFactoryImpl( - WasmEdge_ImportObjectContext *ImpObj, std::shared_ptr instance_env_factory, std::shared_ptr header_repo, std::shared_ptr changes_tracker) : instance_env_factory_{std::move(instance_env_factory)}, header_repo_{std::move(header_repo)}, - changes_tracker_{std::move(changes_tracker)}, - imp_obj_{ImpObj} { + changes_tracker_{std::move(changes_tracker)} { BOOST_ASSERT(instance_env_factory_ != nullptr); BOOST_ASSERT(header_repo_ != nullptr); BOOST_ASSERT(changes_tracker_ != nullptr); @@ -82,8 +72,8 @@ namespace kagome::runtime::wasmedge { const std::vector &runtime_code) const { auto env_factory = std::make_shared( std::make_shared(runtime_code), - std::make_shared( - imp_obj_, runtime_code, instance_env_factory_), + std::make_shared(runtime_code, + instance_env_factory_), header_repo_); auto executor = std::make_unique(header_repo_, env_factory); return std::make_unique( diff --git a/core/runtime/wasmedge/core_api_factory_impl.hpp b/core/runtime/wasmedge/core_api_factory_impl.hpp index ee052c26df..40908329aa 100644 --- a/core/runtime/wasmedge/core_api_factory_impl.hpp +++ b/core/runtime/wasmedge/core_api_factory_impl.hpp @@ -27,8 +27,6 @@ namespace kagome::runtime { class RuntimeEnvironmentFactory; } // namespace kagome::runtime -struct WasmEdge_ImportObjectContext; - namespace kagome::runtime::wasmedge { class InstanceEnvironmentFactory; @@ -39,7 +37,6 @@ namespace kagome::runtime::wasmedge { public std::enable_shared_from_this { public: CoreApiFactoryImpl( - WasmEdge_ImportObjectContext *ImpObj, std::shared_ptr instance_env_factory, std::shared_ptr header_repo, std::shared_ptr changes_tracker); @@ -52,7 +49,6 @@ namespace kagome::runtime::wasmedge { std::shared_ptr instance_env_factory_; std::shared_ptr header_repo_; std::shared_ptr changes_tracker_; - WasmEdge_ImportObjectContext *imp_obj_; }; } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/instance_environment_factory.cpp b/core/runtime/wasmedge/instance_environment_factory.cpp index 98de8500ad..356b6dd755 100644 --- a/core/runtime/wasmedge/instance_environment_factory.cpp +++ b/core/runtime/wasmedge/instance_environment_factory.cpp @@ -42,12 +42,11 @@ namespace kagome::runtime::wasmedge { } WasmedgeInstanceEnvironment InstanceEnvironmentFactory::make() const { - auto imp_obj = WasmEdge_VMGetImportModuleContext(vm_, WasmEdge_HostRegistration_Wasi); memory_provider_->resetMemory(0); auto new_storage_provider = std::make_shared(storage_); auto core_factory = std::make_shared( - imp_obj, shared_from_this(), block_header_repo_, changes_tracker_); + shared_from_this(), block_header_repo_, changes_tracker_); auto host_api = std::shared_ptr(host_api_factory_->make( core_factory, memory_provider_, new_storage_provider)); return WasmedgeInstanceEnvironment{ diff --git a/core/runtime/wasmedge/module_factory_impl.cpp b/core/runtime/wasmedge/module_factory_impl.cpp new file mode 100644 index 0000000000..5ea8b16811 --- /dev/null +++ b/core/runtime/wasmedge/module_factory_impl.cpp @@ -0,0 +1,36 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wasmedge/module_factory_impl.hpp" + +#include "host_api/host_api_factory.hpp" +#include "runtime/wasmedge/memory_provider.hpp" +#include "runtime/wasmedge/core_api_factory_impl.hpp" +#include "runtime/wasmedge/instance_environment_factory.hpp" +#include "runtime/wasmedge/module_impl.hpp" +#include "runtime/common/trie_storage_provider_impl.hpp" + +namespace kagome::runtime::wasmedge { + + ModuleFactoryImpl::ModuleFactoryImpl( + std::shared_ptr env_factory, + std::shared_ptr storage) + : env_factory_{std::move(env_factory)}, storage_{std::move(storage)} { + BOOST_ASSERT(env_factory_ != nullptr); + BOOST_ASSERT(storage_ != nullptr); + } + + outcome::result> ModuleFactoryImpl::make( + storage::trie::RootHash const &state, + gsl::span code) const { + std::vector code_vec{code.begin(), code.end()}; + auto res = ModuleImpl::createFromCode(code_vec, env_factory_); + if (res.has_value()) { + return std::unique_ptr(std::move(res.value())); + } + return res.error(); + } + +} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_factory_impl.hpp b/core/runtime/wasmedge/module_factory_impl.hpp new file mode 100644 index 0000000000..7189290aeb --- /dev/null +++ b/core/runtime/wasmedge/module_factory_impl.hpp @@ -0,0 +1,51 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_FACTORY_IMPL_HPP +#define KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_FACTORY_IMPL_HPP + +#include "runtime/module_factory.hpp" + +namespace kagome::runtime { + class TrieStorageProvider; +} + +namespace kagome::host_api { + class HostApiFactory; +} + +namespace kagome::storage::trie { + class TrieStorage; +} + +namespace kagome::blockchain { + class BlockHeaderRepository; +} + +namespace kagome::storage::changes_trie { + class ChangesTracker; +} + +namespace kagome::runtime::wasmedge { + + class InstanceEnvironmentFactory; + + class ModuleFactoryImpl final : public ModuleFactory { + public: + ModuleFactoryImpl(std::shared_ptr env_factory, + std::shared_ptr storage); + + outcome::result> make( + storage::trie::RootHash const &state, + gsl::span code) const override; + + private: + std::shared_ptr env_factory_; + std::shared_ptr storage_; + }; + +} // namespace kagome::runtime::wasmedge + +#endif // KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_FACTORY_IMPL_HPP diff --git a/core/runtime/wasmedge/module_impl.cpp b/core/runtime/wasmedge/module_impl.cpp new file mode 100644 index 0000000000..907621952d --- /dev/null +++ b/core/runtime/wasmedge/module_impl.cpp @@ -0,0 +1,50 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wasmedge/module_impl.hpp" + +#include + +#include + +#include "common/mp_utils.hpp" +#include "runtime/wasmedge/instance_environment_factory.hpp" +#include "runtime/wasmedge/memory_provider.hpp" +#include "runtime/wasmedge/module_instance_impl.hpp" +#include "storage/trie/polkadot_trie/trie_error.hpp" + +namespace kagome::runtime::wasmedge { + + ModuleImpl::ModuleImpl( + WasmEdge_ASTModuleContext *ast_ctx, + std::shared_ptr env_factory) + : env_factory_{std::move(env_factory)}, ast_ctx_{ast_ctx} { + BOOST_ASSERT(env_factory_ != nullptr); + BOOST_ASSERT(ast_ctx_ != nullptr); + } + + outcome::result> ModuleImpl::createFromCode( + const std::vector &code, + std::shared_ptr env_factory) { + auto log = log::createLogger("wasm_module", "wasmedge"); + + auto *LoaderCtx = WasmEdge_LoaderCreate(nullptr); + WasmEdge_ASTModuleContext *ASTCtx = nullptr; + auto Res = WasmEdge_LoaderParseFromBuffer( + LoaderCtx, &ASTCtx, code.data(), code.size()); + + std::unique_ptr wasm_module_impl( + new ModuleImpl(ASTCtx, std::move(env_factory))); + return wasm_module_impl; + } + + outcome::result> ModuleImpl::instantiate() + const { + auto env = env_factory_->make(); + return std::make_unique( + std::move(env.env), shared_from_this(), env.vm); + } + +} // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_impl.hpp b/core/runtime/wasmedge/module_impl.hpp new file mode 100644 index 0000000000..2a0251484c --- /dev/null +++ b/core/runtime/wasmedge/module_impl.hpp @@ -0,0 +1,63 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_IMPL +#define KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_IMPL + +#include "runtime/module.hpp" + +#include "common/buffer.hpp" +#include "runtime/trie_storage_provider.hpp" + +namespace kagome::storage::trie { + class EphemeralTrieBatch; +} + +struct WasmEdge_ASTModuleContext; + +namespace kagome::runtime::wasmedge { + + class InstanceEnvironmentFactory; + + /** + * Stores a wasm::Module and a wasm::Module instance which contains the module + * and the provided runtime external interface + */ + class ModuleImpl final : public runtime::Module, + public std::enable_shared_from_this { + public: + static constexpr auto kDefaultHeappages = 1024; + enum class Error { EMPTY_STATE_CODE = 1, INVALID_STATE_CODE }; + + ModuleImpl(ModuleImpl &&) = default; + ModuleImpl &operator=(ModuleImpl &&) = default; + + ModuleImpl(const ModuleImpl &) = delete; + ModuleImpl &operator=(const ModuleImpl &) = delete; + + ~ModuleImpl() override = default; + + static outcome::result> createFromCode( + const std::vector &code, + std::shared_ptr env_factory_); + + outcome::result> instantiate() + const override; + + WasmEdge_ASTModuleContext* ast() const { + return ast_ctx_; + } + + private: + ModuleImpl(WasmEdge_ASTModuleContext *ast_ctx, + std::shared_ptr env_factory); + + std::shared_ptr env_factory_; + WasmEdge_ASTModuleContext *ast_ctx_; + }; + +} // namespace kagome::runtime::wasmedge + +#endif // KAGOME_CORE_RUNTIME_WASMEDGE_MODULE_IMPL diff --git a/core/runtime/wasmedge/module_instance_impl.cpp b/core/runtime/wasmedge/module_instance_impl.cpp index bf1a08a02d..7a44032081 100644 --- a/core/runtime/wasmedge/module_instance_impl.cpp +++ b/core/runtime/wasmedge/module_instance_impl.cpp @@ -6,24 +6,21 @@ #include "runtime/wasmedge/module_instance_impl.hpp" #include "host_api/host_api.hpp" +#include "runtime/wasmedge/module_impl.hpp" #include namespace kagome::runtime::wasmedge { ModuleInstanceImpl::ModuleInstanceImpl(InstanceEnvironment &&env, - WasmEdge_ASTModuleContext *parent, - WasmEdge_ImportObjectContext *rei) - + std::shared_ptr parent, + WasmEdge_VMContext *rei) : env_{std::move(env)}, + rei_{rei}, logger_{log::createLogger("ModuleInstance", "wasmedge")} { interpreter_ = WasmEdge_InterpreterCreate(nullptr, nullptr); - store_ = WasmEdge_StoreCreate(); - auto ModName = WasmEdge_StringCreateByCString("ext"); - auto Res = WasmEdge_InterpreterRegisterModule( - interpreter_, store_, parent, ModName); - Res = WasmEdge_InterpreterRegisterImport(interpreter_, store_, rei); - WasmEdge_StringDelete(ModName); + auto store = WasmEdge_VMGetStoreContext(rei_); + WasmEdge_InterpreterInstantiate(interpreter_, store, parent->ast()); } outcome::result ModuleInstanceImpl::callExportFunction( @@ -32,8 +29,9 @@ namespace kagome::runtime::wasmedge { WasmEdge_Value Params[2] = {WasmEdge_ValueGenI32(args.ptr), WasmEdge_ValueGenI32(args.size)}; WasmEdge_Value Returns[1]; + auto store = WasmEdge_VMGetStoreContext(rei_); auto Res = WasmEdge_InterpreterInvoke( - interpreter_, store_, FuncName, Params, 2, Returns, 1); + interpreter_, store, FuncName, Params, 2, Returns, 1); WasmEdge_StringDelete(FuncName); return PtrSize{WasmEdge_ValueGetI64(Returns[0])}; } @@ -41,7 +39,8 @@ namespace kagome::runtime::wasmedge { outcome::result> ModuleInstanceImpl::getGlobal( std::string_view name) const { auto GlobalName = WasmEdge_StringCreateByCString(name.data()); - auto res = WasmEdge_StoreFindGlobal(store_, GlobalName); + auto store = WasmEdge_VMGetStoreContext(rei_); + auto res = WasmEdge_StoreFindGlobal(store, GlobalName); auto type = WasmEdge_GlobalInstanceGetValType(res); auto val = WasmEdge_GlobalInstanceGetValue(res); WasmEdge_StringDelete(GlobalName); diff --git a/core/runtime/wasmedge/module_instance_impl.hpp b/core/runtime/wasmedge/module_instance_impl.hpp index 09cdf38d22..fab7bdbe47 100644 --- a/core/runtime/wasmedge/module_instance_impl.hpp +++ b/core/runtime/wasmedge/module_instance_impl.hpp @@ -10,18 +10,18 @@ #include "log/logger.hpp" -struct WasmEdge_ASTModuleContext; -struct WasmEdge_ImportObjectContext; struct WasmEdge_InterpreterContext; -struct WasmEdge_StoreContext; +struct WasmEdge_VMContext; namespace kagome::runtime::wasmedge { + class ModuleImpl; + class ModuleInstanceImpl final : public ModuleInstance { public: ModuleInstanceImpl(InstanceEnvironment &&env, - WasmEdge_ASTModuleContext *parent, - WasmEdge_ImportObjectContext *rei); + std::shared_ptr parent, + WasmEdge_VMContext *rei); outcome::result callExportFunction(std::string_view name, PtrSize args) const override; @@ -35,7 +35,7 @@ namespace kagome::runtime::wasmedge { private: InstanceEnvironment env_; WasmEdge_InterpreterContext *interpreter_; - WasmEdge_StoreContext *store_; + WasmEdge_VMContext* rei_; log::Logger logger_; }; From 7a84fe8b32bab4e5beabe1db2d75c6c0003c702d Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Wed, 22 Sep 2021 12:16:30 +0300 Subject: [PATCH 10/43] update wasmedge, remove WAVM temporarily --- cmake/dependencies.cmake | 6 +- core/injector/CMakeLists.txt | 2 +- core/injector/application_injector.cpp | 116 +++++++++--------- core/log/configurator.cpp | 2 + core/runtime/CMakeLists.txt | 2 +- core/runtime/executor.hpp | 3 +- core/runtime/memory.hpp | 2 +- .../wasmedge/instance_environment_factory.cpp | 37 +++--- .../wasmedge/instance_environment_factory.hpp | 1 - core/runtime/wasmedge/memory_impl.cpp | 6 +- core/runtime/wasmedge/memory_provider.cpp | 2 +- core/runtime/wasmedge/module_impl.cpp | 9 +- core/runtime/wasmedge/module_impl.hpp | 7 +- .../runtime/wasmedge/module_instance_impl.cpp | 23 ++-- .../runtime/wasmedge/module_instance_impl.hpp | 3 +- core/runtime/wasmedge/register_host_api.hpp | 55 +++++++-- 16 files changed, 163 insertions(+), 113 deletions(-) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index eb39e7d974..d8909559fc 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -70,9 +70,9 @@ find_package(libsecp256k1 CONFIG REQUIRED) hunter_add_package(scale) find_package(scale CONFIG REQUIRED) -hunter_add_package(wavm) -find_package(LLVM CONFIG REQUIRED) -find_package(WAVM CONFIG REQUIRED) +# hunter_add_package(wavm) +# find_package(LLVM CONFIG REQUIRED) +# find_package(WAVM CONFIG REQUIRED) hunter_add_package(zstd) find_package(zstd CONFIG REQUIRED) diff --git a/core/injector/CMakeLists.txt b/core/injector/CMakeLists.txt index a48fcb63ea..e8b56dad02 100644 --- a/core/injector/CMakeLists.txt +++ b/core/injector/CMakeLists.txt @@ -8,7 +8,7 @@ add_library(application_injector ) target_link_libraries(application_injector Boost::Boost.DI - runtime_wavm + # runtime_wavm account_nonce_api api_jrpc_server api_transport diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 22210ac11c..39a158d9ae 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -117,15 +117,15 @@ #include "runtime/wasmedge/instance_environment_factory.hpp" #include "runtime/wasmedge/memory_provider.hpp" #include "runtime/wasmedge/module_factory_impl.hpp" -#include "runtime/wavm/compartment_wrapper.hpp" -#include "runtime/wavm/core_api_factory_impl.hpp" -#include "runtime/wavm/instance_environment_factory.hpp" -#include "runtime/wavm/intrinsics/intrinsic_functions.hpp" -#include "runtime/wavm/intrinsics/intrinsic_module.hpp" -#include "runtime/wavm/intrinsics/intrinsic_module_instance.hpp" -#include "runtime/wavm/intrinsics/intrinsic_resolver_impl.hpp" -#include "runtime/wavm/module.hpp" -#include "runtime/wavm/module_factory_impl.hpp" +// #include "runtime/wavm/compartment_wrapper.hpp" +// #include "runtime/wavm/core_api_factory_impl.hpp" +// #include "runtime/wavm/instance_environment_factory.hpp" +// #include "runtime/wavm/intrinsics/intrinsic_functions.hpp" +// #include "runtime/wavm/intrinsics/intrinsic_module.hpp" +// #include "runtime/wavm/intrinsics/intrinsic_module_instance.hpp" +// #include "runtime/wavm/intrinsics/intrinsic_resolver_impl.hpp" +// #include "runtime/wavm/module.hpp" +// #include "runtime/wavm/module_factory_impl.hpp" #include "storage/changes_trie/impl/storage_changes_tracker_impl.hpp" #include "storage/database_error.hpp" #include "storage/leveldb/leveldb.hpp" @@ -728,46 +728,46 @@ namespace { return initialized.value(); } - template - auto makeWavmInjector( - application::AppConfiguration::RuntimeExecutionMethod method, - Ts &&... args) { - return di::make_injector( - di::bind.template to( - [](const auto &injector) { - static auto compartment = - std::make_shared( - "Runtime Compartment"); - return compartment; - }), - di::bind.template to( - [](const auto &injector) { - static std::shared_ptr module = - [&injector]() { - auto compartment = injector.template create< - sptr>(); - auto module = - std::make_unique( - compartment); - runtime::wavm::registerHostApiMethods(*module); - - return module; - }(); - return module; - }), - di::bind.template to( - [](const auto &injector) { - static std::shared_ptr - instance = [&injector]() { - auto module = injector.template create< - sptr>(); - return module->instantiate(); - }(); - return instance; - }), - di::bind.template to(), - std::forward(args)...); - } + // template + // auto makeWavmInjector( + // application::AppConfiguration::RuntimeExecutionMethod method, + // Ts &&... args) { + // return di::make_injector( + // di::bind.template to( + // [](const auto &injector) { + // static auto compartment = + // std::make_shared( + // "Runtime Compartment"); + // return compartment; + // }), + // di::bind.template to( + // [](const auto &injector) { + // static std::shared_ptr module = + // [&injector]() { + // auto compartment = injector.template create< + // sptr>(); + // auto module = + // std::make_unique( + // compartment); + // runtime::wavm::registerHostApiMethods(*module); + + // return module; + // }(); + // return module; + // }), + // di::bind.template to( + // [](const auto &injector) { + // static std::shared_ptr + // instance = [&injector]() { + // auto module = injector.template create< + // sptr>(); + // return module->instantiate(); + // }(); + // return instance; + // }), + // di::bind.template to(), + // std::forward(args)...); + // } template auto makeBinaryenInjector( @@ -794,7 +794,7 @@ namespace { template auto choose_runtime_implementation( Injector const &injector, @@ -806,9 +806,9 @@ namespace { case RuntimeExecutionMethod::Interpret: return std::static_pointer_cast( injector.template create>()); - case RuntimeExecutionMethod::Compile: - return std::static_pointer_cast( - injector.template create>()); + // case RuntimeExecutionMethod::Compile: + // return std::static_pointer_cast( + // injector.template create>()); case RuntimeExecutionMethod::WasmEdge: return std::static_pointer_cast( injector.template create>()); @@ -858,7 +858,7 @@ namespace { [](auto const &injector) { return get_runtime_upgrade_tracker(injector); }), - makeWavmInjector(method), + // makeWavmInjector(method), makeBinaryenInjector(method), di::bind.template to(), di::bind.template to( @@ -866,16 +866,16 @@ namespace { return choose_runtime_implementation< runtime::CoreApiFactory, runtime::binaryen::CoreApiFactoryImpl, - runtime::wasmedge::CoreApiFactoryImpl, - runtime::wavm::CoreApiFactoryImpl>(injector, method); + runtime::wasmedge::CoreApiFactoryImpl/*, + runtime::wavm::CoreApiFactoryImpl*/>(injector, method); }), di::bind.template to( [method](const auto &injector) { return choose_runtime_implementation< runtime::ModuleFactory, runtime::binaryen::ModuleFactoryImpl, - runtime::wasmedge::ModuleFactoryImpl, - runtime::wavm::ModuleFactoryImpl>(injector, method); + runtime::wasmedge::ModuleFactoryImpl/*, + runtime::wavm::ModuleFactoryImpl*/>(injector, method); }), di::bind.template to([](const auto &injector) { static std::optional> initialized; diff --git a/core/log/configurator.cpp b/core/log/configurator.cpp index b1393e93d8..13aa393cd5 100644 --- a/core/log/configurator.cpp +++ b/core/log/configurator.cpp @@ -55,6 +55,7 @@ namespace kagome::log { children: - name: voting_round - name: runtime + level: trace children: - name: runtime_api - name: host_api @@ -68,6 +69,7 @@ namespace kagome::log { - name: misc_extension - name: binaryen - name: wavm + - name: wasmedge - name: metrics - name: network children: diff --git a/core/runtime/CMakeLists.txt b/core/runtime/CMakeLists.txt index 93058aae0e..7754223070 100644 --- a/core/runtime/CMakeLists.txt +++ b/core/runtime/CMakeLists.txt @@ -6,7 +6,7 @@ add_subdirectory(common) add_subdirectory(binaryen) add_subdirectory(wasmedge) -add_subdirectory(wavm) +# add_subdirectory(wavm) add_library(executor INTERFACE) target_link_libraries(executor INTERFACE Boost::boost blob runtime_environment_factory) diff --git a/core/runtime/executor.hpp b/core/runtime/executor.hpp index d1310b25b6..7a03787f4a 100644 --- a/core/runtime/executor.hpp +++ b/core/runtime/executor.hpp @@ -194,7 +194,8 @@ namespace kagome::runtime { if constexpr (std::is_void_v) { return outcome::success(); } else { - return scale::decode(memory.loadN(result.ptr, result.size)); + auto buf = memory.loadN(result.ptr, result.size); + return scale::decode(buf); } } diff --git a/core/runtime/memory.hpp b/core/runtime/memory.hpp index d0b890ec96..4beea1cb8f 100644 --- a/core/runtime/memory.hpp +++ b/core/runtime/memory.hpp @@ -19,7 +19,7 @@ namespace kagome::runtime { constexpr inline size_t kInitialMemorySize = []() { using kagome::common::literals::operator""_MB; - return 2_MB; + return 5_MB; }(); // according to $3.1.2.1 in the Polkadot Host Spec diff --git a/core/runtime/wasmedge/instance_environment_factory.cpp b/core/runtime/wasmedge/instance_environment_factory.cpp index 356b6dd755..bf01752e12 100644 --- a/core/runtime/wasmedge/instance_environment_factory.cpp +++ b/core/runtime/wasmedge/instance_environment_factory.cpp @@ -12,8 +12,6 @@ #include "runtime/wasmedge/memory_provider.hpp" #include "runtime/wasmedge/register_host_api.hpp" -#include - namespace kagome::runtime::wasmedge { InstanceEnvironmentFactory::InstanceEnvironmentFactory( @@ -29,31 +27,36 @@ namespace kagome::runtime::wasmedge { BOOST_ASSERT(host_api_factory_); BOOST_ASSERT(block_header_repo_); BOOST_ASSERT(changes_tracker_); - - vm_ = WasmEdge_VMCreate(NULL, NULL); - WasmEdge_String ExportName = WasmEdge_StringCreateByCString("ext"); - WasmEdge_ImportObjectContext *ImpObj = - WasmEdge_ImportObjectCreate(ExportName, NULL); - WasmEdge_StringDelete(ExportName); - register_host_api(ImpObj); - memory_provider_ = std::make_shared(); - memory_provider_->setExternalInterface(ImpObj); - WasmEdge_VMRegisterModuleFromImport(vm_, ImpObj); } WasmedgeInstanceEnvironment InstanceEnvironmentFactory::make() const { - memory_provider_->resetMemory(0); auto new_storage_provider = std::make_shared(storage_); auto core_factory = std::make_shared( shared_from_this(), block_header_repo_, changes_tracker_); + auto memory_provider = std::make_shared(); auto host_api = std::shared_ptr(host_api_factory_->make( - core_factory, memory_provider_, new_storage_provider)); + core_factory, memory_provider, new_storage_provider)); + + // WasmEdge_ImportObjectContext *ImpObj = + // WasmEdge_VMGetImportModuleContext(vm_, WasmEdge_HostRegistration_Wasi); + // if (ImpObj) { + // register_host_api(ImpObj); + // memory_provider->setExternalInterface(ImpObj); + // } else { + // WasmEdge_String ExportName = WasmEdge_StringCreateByCString("env"); + // WasmEdge_ImportObjectContext *NewImpObj = + // WasmEdge_ImportObjectCreate(ExportName, host_api.get()); + // WasmEdge_StringDelete(ExportName); + // register_host_api(NewImpObj); + // memory_provider->setExternalInterface(NewImpObj); + // WasmEdge_VMRegisterModuleFromImport(vm_, NewImpObj); + // } + return WasmedgeInstanceEnvironment{ - InstanceEnvironment{memory_provider_, + InstanceEnvironment{memory_provider, std::move(new_storage_provider), std::move(host_api), - [](auto &) {}}, - vm_}; + [](auto &) {}}}; } } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/instance_environment_factory.hpp b/core/runtime/wasmedge/instance_environment_factory.hpp index a57e25c3b4..05dd2e6368 100644 --- a/core/runtime/wasmedge/instance_environment_factory.hpp +++ b/core/runtime/wasmedge/instance_environment_factory.hpp @@ -32,7 +32,6 @@ namespace kagome::runtime::wasmedge { struct WasmedgeInstanceEnvironment { InstanceEnvironment env; - WasmEdge_VMContext *vm; }; class InstanceEnvironmentFactory final diff --git a/core/runtime/wasmedge/memory_impl.cpp b/core/runtime/wasmedge/memory_impl.cpp index 45d74e5903..6c479e955f 100644 --- a/core/runtime/wasmedge/memory_impl.cpp +++ b/core/runtime/wasmedge/memory_impl.cpp @@ -40,7 +40,7 @@ namespace kagome::runtime::wasmedge { : memory_{memory}, size_{kInitialMemorySize}, allocator_{std::move(allocator)}, - logger_{log::createLogger("Binaryen Memory", "binaryen")} { + logger_{log::createLogger("WasmEdge Memory", "wasmedge")} { resize(size_); } @@ -51,7 +51,7 @@ namespace kagome::runtime::wasmedge { [this](auto new_size) { return resize(new_size); }, [this]() { return size_; }}, kInitialMemorySize, - 5_MB)} {} + 2000000)} {} WasmPointer MemoryImpl::allocate(WasmSize size) { return allocator_->allocate(size); @@ -178,7 +178,7 @@ namespace kagome::runtime::wasmedge { gsl::span value) { const auto size = static_cast(value.size()); BOOST_ASSERT((allocator_->checkAddress(addr, size))); - WasmEdge_Result Res = WasmEdge_MemoryInstanceGetData( + WasmEdge_Result Res = WasmEdge_MemoryInstanceSetData( memory_, const_cast(value.data()), addr, size); } diff --git a/core/runtime/wasmedge/memory_provider.cpp b/core/runtime/wasmedge/memory_provider.cpp index 15200546fc..c48964c162 100644 --- a/core/runtime/wasmedge/memory_provider.cpp +++ b/core/runtime/wasmedge/memory_provider.cpp @@ -25,7 +25,7 @@ namespace kagome::runtime::wasmedge { void WasmedgeMemoryProvider::setExternalInterface( WasmEdge_ImportObjectContext *imp_obj) { - WasmEdge_Limit MemoryLimit = {.HasMax = false, .Min = 40, .Max = 40}; + WasmEdge_Limit MemoryLimit = {.HasMax = false, .Min = 500, .Max = 500}; mem_ctx_ = WasmEdge_MemoryInstanceCreate(MemoryLimit); WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); WasmEdge_ImportObjectAddMemory(imp_obj, MemoryName, mem_ctx_); diff --git a/core/runtime/wasmedge/module_impl.cpp b/core/runtime/wasmedge/module_impl.cpp index 907621952d..48cac6858b 100644 --- a/core/runtime/wasmedge/module_impl.cpp +++ b/core/runtime/wasmedge/module_impl.cpp @@ -18,11 +18,16 @@ namespace kagome::runtime::wasmedge { ModuleImpl::ModuleImpl( - WasmEdge_ASTModuleContext *ast_ctx, + const WasmEdge_ASTModuleContext *ast_ctx, std::shared_ptr env_factory) : env_factory_{std::move(env_factory)}, ast_ctx_{ast_ctx} { BOOST_ASSERT(env_factory_ != nullptr); BOOST_ASSERT(ast_ctx_ != nullptr); + vm_ = WasmEdge_VMCreate(NULL, NULL); + WasmEdge_String ModuleName = WasmEdge_StringCreateByCString("env"); + auto Res = WasmEdge_VMRegisterModuleFromASTModule(vm_, ModuleName, ast_ctx_); + WasmEdge_StringDelete(ModuleName); + BOOST_ASSERT(vm_ != nullptr); } outcome::result> ModuleImpl::createFromCode( @@ -44,7 +49,7 @@ namespace kagome::runtime::wasmedge { const { auto env = env_factory_->make(); return std::make_unique( - std::move(env.env), shared_from_this(), env.vm); + std::move(env.env), this, vm_); } } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_impl.hpp b/core/runtime/wasmedge/module_impl.hpp index 2a0251484c..3b6eee62ce 100644 --- a/core/runtime/wasmedge/module_impl.hpp +++ b/core/runtime/wasmedge/module_impl.hpp @@ -46,16 +46,17 @@ namespace kagome::runtime::wasmedge { outcome::result> instantiate() const override; - WasmEdge_ASTModuleContext* ast() const { + const WasmEdge_ASTModuleContext* ast() const { return ast_ctx_; } private: - ModuleImpl(WasmEdge_ASTModuleContext *ast_ctx, + ModuleImpl(const WasmEdge_ASTModuleContext *ast_ctx, std::shared_ptr env_factory); std::shared_ptr env_factory_; - WasmEdge_ASTModuleContext *ast_ctx_; + const WasmEdge_ASTModuleContext *ast_ctx_; + WasmEdge_VMContext* vm_; }; } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_instance_impl.cpp b/core/runtime/wasmedge/module_instance_impl.cpp index 7a44032081..a9697402ba 100644 --- a/core/runtime/wasmedge/module_instance_impl.cpp +++ b/core/runtime/wasmedge/module_instance_impl.cpp @@ -13,14 +13,19 @@ namespace kagome::runtime::wasmedge { ModuleInstanceImpl::ModuleInstanceImpl(InstanceEnvironment &&env, - std::shared_ptr parent, + const ModuleImpl* parent, WasmEdge_VMContext *rei) : env_{std::move(env)}, rei_{rei}, + parent_{parent}, logger_{log::createLogger("ModuleInstance", "wasmedge")} { - interpreter_ = WasmEdge_InterpreterCreate(nullptr, nullptr); - auto store = WasmEdge_VMGetStoreContext(rei_); - WasmEdge_InterpreterInstantiate(interpreter_, store, parent->ast()); + WasmEdge_ImportObjectContext *ImpObj = + WasmEdge_VMGetImportModuleContext(vm_, WasmEdge_HostRegistration_Wasi); + register_host_api(ImpObj); + dynamic_cast(env.memory_provider.get())->setExternalInterface(ImpObj); + // interpreter_ = WasmEdge_InterpreterCreate(nullptr, nullptr); + // auto store = WasmEdge_VMGetStoreContext(rei_); + // WasmEdge_InterpreterInstantiate(interpreter_, store, parent->ast()); } outcome::result ModuleInstanceImpl::callExportFunction( @@ -29,11 +34,13 @@ namespace kagome::runtime::wasmedge { WasmEdge_Value Params[2] = {WasmEdge_ValueGenI32(args.ptr), WasmEdge_ValueGenI32(args.size)}; WasmEdge_Value Returns[1]; - auto store = WasmEdge_VMGetStoreContext(rei_); - auto Res = WasmEdge_InterpreterInvoke( - interpreter_, store, FuncName, Params, 2, Returns, 1); + // auto store = WasmEdge_VMGetStoreContext(rei_); + auto Res = WasmEdge_VMRunWasmFromASTModule(rei_, parent_->ast(), FuncName, Params, 2, Returns, 1); + // auto Res = WasmEdge_InterpreterInvoke( + // interpreter_, store, FuncName, Params, 2, Returns, 1); WasmEdge_StringDelete(FuncName); - return PtrSize{WasmEdge_ValueGetI64(Returns[0])}; + auto i = WasmEdge_ValueGetI64(Returns[0]); + return PtrSize{i}; } outcome::result> ModuleInstanceImpl::getGlobal( diff --git a/core/runtime/wasmedge/module_instance_impl.hpp b/core/runtime/wasmedge/module_instance_impl.hpp index fab7bdbe47..bbdb79f18c 100644 --- a/core/runtime/wasmedge/module_instance_impl.hpp +++ b/core/runtime/wasmedge/module_instance_impl.hpp @@ -20,7 +20,7 @@ namespace kagome::runtime::wasmedge { class ModuleInstanceImpl final : public ModuleInstance { public: ModuleInstanceImpl(InstanceEnvironment &&env, - std::shared_ptr parent, + const ModuleImpl* parent, WasmEdge_VMContext *rei); outcome::result callExportFunction(std::string_view name, PtrSize args) const override; @@ -36,6 +36,7 @@ namespace kagome::runtime::wasmedge { InstanceEnvironment env_; WasmEdge_InterpreterContext *interpreter_; WasmEdge_VMContext* rei_; + const ModuleImpl* parent_; log::Logger logger_; }; diff --git a/core/runtime/wasmedge/register_host_api.hpp b/core/runtime/wasmedge/register_host_api.hpp index dbe0a952e8..220741c279 100644 --- a/core/runtime/wasmedge/register_host_api.hpp +++ b/core/runtime/wasmedge/register_host_api.hpp @@ -6,6 +6,44 @@ #include #include +template +struct MakeSigned { + using type = typename std:: + conditional_t, typename std::make_signed_t, T>; +}; +template <> +struct MakeSigned { + using type = void; +}; + +template +struct CommonInt { + using type = __int128; +}; + +template <> +struct CommonInt<4> { + using type = int32_t; +}; + +template <> +struct CommonInt<8> { + using type = int64_t; +}; + +template +struct MakeCommon; + +template +struct MakeCommon>> { + using type = typename CommonInt::type; +}; + +template +struct MakeCommon>> { + using type = T; +}; + template WasmEdge_Value (*fromType())(const T) { return nullptr; @@ -57,16 +95,6 @@ __int128 (*toType<__int128>())(const WasmEdge_Value) { return WasmEdge_ValueGetV128; } -template -struct MakeSigned { - using type = typename std:: - conditional_t, typename std::make_signed_t, T>; -}; -template <> -struct MakeSigned { - using type = void; -}; - template WasmEdge_ValType type() { return WasmEdge_ValType_V128; @@ -119,7 +147,7 @@ WasmEdge_Result fun(void *Data, void, typename HostApiFuncRet::Ret>) { auto res = internal_fun(static_cast(Data), In, tup, indices); - Out[0] = fromType()(res); + Out[0] = fromType::type>()(res); } else { internal_fun(static_cast(Data), In, tup, indices); } @@ -205,7 +233,7 @@ void registerHostApiFunc(const std::string &name, namespace { using kagome::host_api::HostApi; using kagome::host_api::HostApiImpl; -} +} // namespace inline void register_host_api(WasmEdge_ImportObjectContext *ImpObj) { REGISTER_HOST_API_FUNC( @@ -240,6 +268,7 @@ inline void register_host_api(WasmEdge_ImportObjectContext *ImpObj) { REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_free_version_1, ImpObj); REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_malloc_version_1, ImpObj); REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_hex_version_1, ImpObj); + REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_num_version_1, ImpObj); REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_utf8_version_1, ImpObj); REGISTER_HOST_API_FUNC( HostApiImpl, ext_misc_runtime_version_version_1, ImpObj); @@ -265,6 +294,8 @@ inline void register_host_api(WasmEdge_ImportObjectContext *ImpObj) { REGISTER_HOST_API_FUNC( HostApiImpl, ext_storage_changes_root_version_1, ImpObj); REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_clear_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_storage_clear_prefix_version_1, ImpObj); REGISTER_HOST_API_FUNC( HostApiImpl, ext_storage_clear_prefix_version_2, ImpObj); REGISTER_HOST_API_FUNC( From d72b0a76b8cf4311a927e0de684ebdfbf5ac5fb7 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Wed, 29 Sep 2021 16:19:50 +0300 Subject: [PATCH 11/43] implement compiling wasmedge --- core/runtime/wasmedge/module_factory_impl.cpp | 1 - core/runtime/wasmedge/module_factory_impl.hpp | 1 - core/runtime/wasmedge/module_impl.cpp | 34 +- core/runtime/wasmedge/module_impl.hpp | 3 +- .../runtime/wasmedge/module_instance_impl.cpp | 24 +- .../runtime/wasmedge/module_instance_impl.hpp | 8 +- core/runtime/wasmedge/register_host_api.hpp | 368 +++++++++--------- 7 files changed, 229 insertions(+), 210 deletions(-) diff --git a/core/runtime/wasmedge/module_factory_impl.cpp b/core/runtime/wasmedge/module_factory_impl.cpp index 5ea8b16811..227b3d4845 100644 --- a/core/runtime/wasmedge/module_factory_impl.cpp +++ b/core/runtime/wasmedge/module_factory_impl.cpp @@ -23,7 +23,6 @@ namespace kagome::runtime::wasmedge { } outcome::result> ModuleFactoryImpl::make( - storage::trie::RootHash const &state, gsl::span code) const { std::vector code_vec{code.begin(), code.end()}; auto res = ModuleImpl::createFromCode(code_vec, env_factory_); diff --git a/core/runtime/wasmedge/module_factory_impl.hpp b/core/runtime/wasmedge/module_factory_impl.hpp index 7189290aeb..9c7a12e102 100644 --- a/core/runtime/wasmedge/module_factory_impl.hpp +++ b/core/runtime/wasmedge/module_factory_impl.hpp @@ -38,7 +38,6 @@ namespace kagome::runtime::wasmedge { std::shared_ptr storage); outcome::result> make( - storage::trie::RootHash const &state, gsl::span code) const override; private: diff --git a/core/runtime/wasmedge/module_impl.cpp b/core/runtime/wasmedge/module_impl.cpp index 48cac6858b..27caf194ca 100644 --- a/core/runtime/wasmedge/module_impl.cpp +++ b/core/runtime/wasmedge/module_impl.cpp @@ -5,6 +5,7 @@ #include "runtime/wasmedge/module_impl.hpp" +#include #include #include @@ -23,11 +24,11 @@ namespace kagome::runtime::wasmedge { : env_factory_{std::move(env_factory)}, ast_ctx_{ast_ctx} { BOOST_ASSERT(env_factory_ != nullptr); BOOST_ASSERT(ast_ctx_ != nullptr); - vm_ = WasmEdge_VMCreate(NULL, NULL); - WasmEdge_String ModuleName = WasmEdge_StringCreateByCString("env"); - auto Res = WasmEdge_VMRegisterModuleFromASTModule(vm_, ModuleName, ast_ctx_); - WasmEdge_StringDelete(ModuleName); - BOOST_ASSERT(vm_ != nullptr); + // vm_ = WasmEdge_VMCreate(NULL, NULL); + // WasmEdge_String ModuleName = WasmEdge_StringCreateByCString("env"); + // auto Res = WasmEdge_VMRegisterModuleFromASTModule(vm_, ModuleName, + // ast_ctx_); WasmEdge_StringDelete(ModuleName); BOOST_ASSERT(vm_ != + // nullptr); } outcome::result> ModuleImpl::createFromCode( @@ -35,21 +36,32 @@ namespace kagome::runtime::wasmedge { std::shared_ptr env_factory) { auto log = log::createLogger("wasm_module", "wasmedge"); - auto *LoaderCtx = WasmEdge_LoaderCreate(nullptr); + auto ConfCtx = WasmEdge_ConfigureCreate(); + WasmEdge_ConfigureCompilerSetOptimizationLevel( + ConfCtx, WasmEdge_CompilerOptimizationLevel_O3); + auto CompilerCtx = WasmEdge_CompilerCreate(ConfCtx); + + std::ofstream ofs; + ofs.open("source.wasm"); + ofs.write(reinterpret_cast(code.data()), code.size()); + ofs.close(); + + WasmEdge_CompilerCompile(CompilerCtx, "source.wasm", "result.wasm.so"); + + auto *LoaderCtx = WasmEdge_LoaderCreate(ConfCtx); WasmEdge_ASTModuleContext *ASTCtx = nullptr; - auto Res = WasmEdge_LoaderParseFromBuffer( - LoaderCtx, &ASTCtx, code.data(), code.size()); + auto Res = + WasmEdge_LoaderParseFromFile(LoaderCtx, &ASTCtx, "result.wasm.so"); std::unique_ptr wasm_module_impl( new ModuleImpl(ASTCtx, std::move(env_factory))); return wasm_module_impl; } - outcome::result> ModuleImpl::instantiate() + outcome::result> ModuleImpl::instantiate() const { auto env = env_factory_->make(); - return std::make_unique( - std::move(env.env), this, vm_); + return std::make_shared(std::move(env.env), this); } } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_impl.hpp b/core/runtime/wasmedge/module_impl.hpp index 3b6eee62ce..c4c7f7c59d 100644 --- a/core/runtime/wasmedge/module_impl.hpp +++ b/core/runtime/wasmedge/module_impl.hpp @@ -43,7 +43,7 @@ namespace kagome::runtime::wasmedge { const std::vector &code, std::shared_ptr env_factory_); - outcome::result> instantiate() + outcome::result> instantiate() const override; const WasmEdge_ASTModuleContext* ast() const { @@ -56,7 +56,6 @@ namespace kagome::runtime::wasmedge { std::shared_ptr env_factory_; const WasmEdge_ASTModuleContext *ast_ctx_; - WasmEdge_VMContext* vm_; }; } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_instance_impl.cpp b/core/runtime/wasmedge/module_instance_impl.cpp index a9697402ba..6da349467d 100644 --- a/core/runtime/wasmedge/module_instance_impl.cpp +++ b/core/runtime/wasmedge/module_instance_impl.cpp @@ -5,24 +5,33 @@ #include "runtime/wasmedge/module_instance_impl.hpp" -#include "host_api/host_api.hpp" +#include "host_api/impl/host_api_impl.hpp" +#include "runtime/wasmedge/memory_provider.hpp" #include "runtime/wasmedge/module_impl.hpp" +#include "runtime/wasmedge/register_host_api.hpp" #include namespace kagome::runtime::wasmedge { ModuleInstanceImpl::ModuleInstanceImpl(InstanceEnvironment &&env, - const ModuleImpl* parent, - WasmEdge_VMContext *rei) + const ModuleImpl *parent) : env_{std::move(env)}, - rei_{rei}, parent_{parent}, logger_{log::createLogger("ModuleInstance", "wasmedge")} { + rei_ = WasmEdge_VMCreate(NULL, NULL); + WasmEdge_String ModuleName = WasmEdge_StringCreateByCString("env"); +// auto Res = WasmEdge_VMRegisterModuleFromASTModule( +// rei_, ModuleName, parent_->ast()); WasmEdge_ImportObjectContext *ImpObj = - WasmEdge_VMGetImportModuleContext(vm_, WasmEdge_HostRegistration_Wasi); + WasmEdge_ImportObjectCreate(ModuleName, env_.host_api.get()); +// WasmEdge_ImportObjectContext *ImpObj = +// WasmEdge_VMGetImportModuleContext(rei_, WasmEdge_HostRegistration_WasmEdge_Process); + WasmEdge_StringDelete(ModuleName); register_host_api(ImpObj); - dynamic_cast(env.memory_provider.get())->setExternalInterface(ImpObj); + dynamic_cast(env_.memory_provider.get()) + ->setExternalInterface(ImpObj); + auto Res = WasmEdge_VMRegisterModuleFromImport(rei_, ImpObj); // interpreter_ = WasmEdge_InterpreterCreate(nullptr, nullptr); // auto store = WasmEdge_VMGetStoreContext(rei_); // WasmEdge_InterpreterInstantiate(interpreter_, store, parent->ast()); @@ -35,7 +44,8 @@ namespace kagome::runtime::wasmedge { WasmEdge_ValueGenI32(args.size)}; WasmEdge_Value Returns[1]; // auto store = WasmEdge_VMGetStoreContext(rei_); - auto Res = WasmEdge_VMRunWasmFromASTModule(rei_, parent_->ast(), FuncName, Params, 2, Returns, 1); + auto Res = WasmEdge_VMRunWasmFromASTModule( + rei_, parent_->ast(), FuncName, Params, 2, Returns, 1); // auto Res = WasmEdge_InterpreterInvoke( // interpreter_, store, FuncName, Params, 2, Returns, 1); WasmEdge_StringDelete(FuncName); diff --git a/core/runtime/wasmedge/module_instance_impl.hpp b/core/runtime/wasmedge/module_instance_impl.hpp index bbdb79f18c..49d1dcd2de 100644 --- a/core/runtime/wasmedge/module_instance_impl.hpp +++ b/core/runtime/wasmedge/module_instance_impl.hpp @@ -19,9 +19,7 @@ namespace kagome::runtime::wasmedge { class ModuleInstanceImpl final : public ModuleInstance { public: - ModuleInstanceImpl(InstanceEnvironment &&env, - const ModuleImpl* parent, - WasmEdge_VMContext *rei); + ModuleInstanceImpl(InstanceEnvironment &&env, const ModuleImpl *parent); outcome::result callExportFunction(std::string_view name, PtrSize args) const override; @@ -35,8 +33,8 @@ namespace kagome::runtime::wasmedge { private: InstanceEnvironment env_; WasmEdge_InterpreterContext *interpreter_; - WasmEdge_VMContext* rei_; - const ModuleImpl* parent_; + WasmEdge_VMContext *rei_; + const ModuleImpl *parent_; log::Logger logger_; }; diff --git a/core/runtime/wasmedge/register_host_api.hpp b/core/runtime/wasmedge/register_host_api.hpp index 220741c279..b7d36258e3 100644 --- a/core/runtime/wasmedge/register_host_api.hpp +++ b/core/runtime/wasmedge/register_host_api.hpp @@ -6,213 +6,215 @@ #include #include -template -struct MakeSigned { - using type = typename std:: - conditional_t, typename std::make_signed_t, T>; -}; -template <> -struct MakeSigned { - using type = void; -}; +namespace { + template + struct MakeSigned { + using type = typename std:: + conditional_t, typename std::make_signed_t, T>; + }; + template <> + struct MakeSigned { + using type = void; + }; -template -struct CommonInt { - using type = __int128; -}; + template + struct CommonInt { + using type = __int128; + }; -template <> -struct CommonInt<4> { - using type = int32_t; -}; + template <> + struct CommonInt<4> { + using type = int32_t; + }; -template <> -struct CommonInt<8> { - using type = int64_t; -}; + template <> + struct CommonInt<8> { + using type = int64_t; + }; -template -struct MakeCommon; + template + struct MakeCommon; -template -struct MakeCommon>> { - using type = typename CommonInt::type; -}; + template + struct MakeCommon>> { + using type = typename CommonInt::type; + }; -template -struct MakeCommon>> { - using type = T; -}; + template + struct MakeCommon>> { + using type = T; + }; -template -WasmEdge_Value (*fromType())(const T) { - return nullptr; -} -template <> -WasmEdge_Value (*fromType())(const int32_t) { - return WasmEdge_ValueGenI32; -} -template <> -WasmEdge_Value (*fromType())(const int64_t) { - return WasmEdge_ValueGenI64; -} -template <> -WasmEdge_Value (*fromType())(const float) { - return WasmEdge_ValueGenF32; -} -template <> -WasmEdge_Value (*fromType())(const double) { - return WasmEdge_ValueGenF64; -} -template <> -WasmEdge_Value (*fromType<__int128>())(const __int128) { - return WasmEdge_ValueGenV128; -} - -template -T (*toType()) -(const WasmEdge_Value) { - return nullptr; -} -template <> -int32_t (*toType())(const WasmEdge_Value) { - return WasmEdge_ValueGetI32; -} -template <> -int64_t (*toType())(const WasmEdge_Value) { - return WasmEdge_ValueGetI64; -} -template <> -float (*toType())(const WasmEdge_Value) { - return WasmEdge_ValueGetF32; -} -template <> -double (*toType())(const WasmEdge_Value) { - return WasmEdge_ValueGetF64; -} -template <> -__int128 (*toType<__int128>())(const WasmEdge_Value) { - return WasmEdge_ValueGetV128; -} - -template -WasmEdge_ValType type() { - return WasmEdge_ValType_V128; -} -template <> -WasmEdge_ValType type() { - return WasmEdge_ValType_I32; -} -template <> -WasmEdge_ValType type() { - return WasmEdge_ValType_I64; -} -template <> -WasmEdge_ValType type() { - return WasmEdge_ValType_F32; -} -template <> -WasmEdge_ValType type() { - return WasmEdge_ValType_F64; -} + template + WasmEdge_Value (*fromType())(const T) { + return nullptr; + } + template <> + WasmEdge_Value (*fromType())(const int32_t) { + return WasmEdge_ValueGenI32; + } + template <> + WasmEdge_Value (*fromType())(const int64_t) { + return WasmEdge_ValueGenI64; + } + template <> + WasmEdge_Value (*fromType())(const float) { + return WasmEdge_ValueGenF32; + } + template <> + WasmEdge_Value (*fromType())(const double) { + return WasmEdge_ValueGenF64; + } + template <> + WasmEdge_Value (*fromType<__int128>())(const __int128) { + return WasmEdge_ValueGenV128; + } -template -auto internal_fun(T *fun, - const WasmEdge_Value *In, - const TP &tup, - std::index_sequence) { - return (fun->*f)(std::get(tup)(In[I])...); -} + template + T (*toType()) + (const WasmEdge_Value) { + return nullptr; + } + template <> + int32_t (*toType())(const WasmEdge_Value) { + return WasmEdge_ValueGetI32; + } + template <> + int64_t (*toType())(const WasmEdge_Value) { + return WasmEdge_ValueGetI64; + } + template <> + float (*toType())(const WasmEdge_Value) { + return WasmEdge_ValueGetF32; + } + template <> + double (*toType())(const WasmEdge_Value) { + return WasmEdge_ValueGetF64; + } + template <> + __int128 (*toType<__int128>())(const WasmEdge_Value) { + return WasmEdge_ValueGetV128; + } -template -struct HostApiFuncRet; -template -struct HostApiFuncRet { - using Ret = R; -}; -template -struct HostApiFuncRet { - using Ret = R; -}; + template + WasmEdge_ValType type() { + return WasmEdge_ValType_V128; + } + template <> + WasmEdge_ValType type() { + return WasmEdge_ValType_I32; + } + template <> + WasmEdge_ValType type() { + return WasmEdge_ValType_I64; + } + template <> + WasmEdge_ValType type() { + return WasmEdge_ValType_F32; + } + template <> + WasmEdge_ValType type() { + return WasmEdge_ValType_F64; + } -/* Host function body definition. */ -template -WasmEdge_Result fun(void *Data, - WasmEdge_MemoryInstanceContext *MemCxt, + template + auto internal_fun(T *fun, const WasmEdge_Value *In, - WasmEdge_Value *Out) { - auto indices = std::index_sequence_for{}; - auto tup = std::make_tuple(toType::type>()...); - if constexpr (not std::is_same_v< - void, - typename HostApiFuncRet::Ret>) { - auto res = internal_fun(static_cast(Data), In, tup, indices); - Out[0] = fromType::type>()(res); - } else { - internal_fun(static_cast(Data), In, tup, indices); + const TP &tup, + std::index_sequence) { + return (fun->*f)(std::get(tup)(In[I])...); } - return WasmEdge_Result_Success; -} -template -struct HostApiFunc; -template -struct HostApiFunc { - using Ret = R; - static WasmEdge_Result (*fun())(void *Data, - WasmEdge_MemoryInstanceContext *MemCxt, - const WasmEdge_Value *In, - WasmEdge_Value *Out) { - return ::fun; + template + struct HostApiFuncRet; + template + struct HostApiFuncRet { + using Ret = R; + }; + template + struct HostApiFuncRet { + using Ret = R; + }; + + /* Host function body definition. */ + template + WasmEdge_Result fun(void *Data, + WasmEdge_MemoryInstanceContext *MemCxt, + const WasmEdge_Value *In, + WasmEdge_Value *Out) { + auto indices = std::index_sequence_for{}; + auto tup = std::make_tuple(toType::type>()...); + if constexpr (not std::is_same_v< + void, + typename HostApiFuncRet::Ret>) { + auto res = internal_fun(static_cast(Data), In, tup, indices); + Out[0] = fromType::type>()(res); + } else { + internal_fun(static_cast(Data), In, tup, indices); + } + return WasmEdge_Result_Success; } - static WasmEdge_FunctionTypeContext *type() { - enum WasmEdge_ValType ParamList[sizeof...(Args)] = { - ::type::type>()...}; + template + struct HostApiFunc; + template + struct HostApiFunc { + using Ret = R; + static WasmEdge_Result (*fun())(void *Data, + WasmEdge_MemoryInstanceContext *MemCxt, + const WasmEdge_Value *In, + WasmEdge_Value *Out) { + return ::fun; + } + + static WasmEdge_FunctionTypeContext *type() { + enum WasmEdge_ValType ParamList[sizeof...(Args)] = { + ::type::type>()...}; + + size_t r = 1; + enum WasmEdge_ValType ReturnListNonVoid[1] = { + ::type::type>()}; + enum WasmEdge_ValType *ReturnList = ReturnListNonVoid; - size_t r = 1; - enum WasmEdge_ValType ReturnListNonVoid[1] = { - ::type::type>()}; - enum WasmEdge_ValType *ReturnList = ReturnListNonVoid; + if constexpr (std::is_same_v) { + r = 0; + ReturnList = nullptr; + } - if constexpr (std::is_same_v) { - r = 0; - ReturnList = nullptr; + return WasmEdge_FunctionTypeCreate( + ParamList, sizeof...(Args), ReturnList, r); } + }; - return WasmEdge_FunctionTypeCreate( - ParamList, sizeof...(Args), ReturnList, r); - } -}; + template + struct HostApiFunc { + using Ret = R; + static WasmEdge_Result (*fun())(void *Data, + WasmEdge_MemoryInstanceContext *MemCxt, + const WasmEdge_Value *In, + WasmEdge_Value *Out) { + return ::fun; + } -template -struct HostApiFunc { - using Ret = R; - static WasmEdge_Result (*fun())(void *Data, - WasmEdge_MemoryInstanceContext *MemCxt, - const WasmEdge_Value *In, - WasmEdge_Value *Out) { - return ::fun; - } + static WasmEdge_FunctionTypeContext *type() { + enum WasmEdge_ValType ParamList[sizeof...(Args)] = { + ::type::type>()...}; - static WasmEdge_FunctionTypeContext *type() { - enum WasmEdge_ValType ParamList[sizeof...(Args)] = { - ::type::type>()...}; + size_t r = 1; + enum WasmEdge_ValType ReturnListNonVoid[1] = { + ::type::type>()}; + enum WasmEdge_ValType *ReturnList = ReturnListNonVoid; - size_t r = 1; - enum WasmEdge_ValType ReturnListNonVoid[1] = { - ::type::type>()}; - enum WasmEdge_ValType *ReturnList = ReturnListNonVoid; + if constexpr (std::is_same_v) { + r = 0; + ReturnList = nullptr; + } - if constexpr (std::is_same_v) { - r = 0; - ReturnList = nullptr; + return WasmEdge_FunctionTypeCreate( + ParamList, sizeof...(Args), ReturnList, r); } - - return WasmEdge_FunctionTypeCreate( - ParamList, sizeof...(Args), ReturnList, r); - } -}; + }; +} // namespace template void registerHostApiFunc(const std::string &name, From 3ca7256334437cd84f2c860021f489d7e4b5ede7 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Thu, 28 Oct 2021 13:27:05 +0300 Subject: [PATCH 12/43] fix --- .../impl/app_configuration_impl.cpp | 2 +- core/host_api/host_api.hpp | 2 +- core/host_api/impl/host_api_impl.cpp | 4 +- core/host_api/impl/host_api_impl.hpp | 42 ++++++------------- core/log/configurator.cpp | 2 +- .../wasmedge/core_api_factory_impl.cpp | 3 +- .../wasmedge/core_api_factory_impl.hpp | 2 +- core/runtime/wasmedge/memory_impl.cpp | 2 +- core/runtime/wasmedge/memory_impl.hpp | 2 +- core/runtime/wasmedge/memory_provider.cpp | 10 +++-- core/runtime/wasmedge/memory_provider.hpp | 5 ++- core/runtime/wasmedge/module_impl.cpp | 23 ++++++---- .../runtime/wasmedge/module_instance_impl.cpp | 15 +++---- .../runtime/wasmedge/module_instance_impl.hpp | 2 +- core/runtime/wasmedge/register_host_api.hpp | 14 +++++++ core/utils/profiler.hpp | 29 +++++++++++-- 16 files changed, 96 insertions(+), 63 deletions(-) diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index 2bd761191f..72bdbda4d3 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -84,7 +84,7 @@ namespace { if (str == "WasmEdge") { return REM::WasmEdge; } - return std::nullopt_t; + return std::nullopt; } std::optional diff --git a/core/host_api/host_api.hpp b/core/host_api/host_api.hpp index f75aebcebb..0a2e945780 100644 --- a/core/host_api/host_api.hpp +++ b/core/host_api/host_api.hpp @@ -370,7 +370,7 @@ namespace kagome::host_api { // -------------------------- Offchain extension --------------------------- /// @copydoc OffchainExtension::ext_offchain_is_validator_version_1 - [[nodiscard]] virtual runtime::WasmI8 + [[nodiscard]] virtual runtime::WasmI32 ext_offchain_is_validator_version_1() = 0; /// @copydoc OffchainExtension::ext_offchain_submit_transaction_version_1 diff --git a/core/host_api/impl/host_api_impl.cpp b/core/host_api/impl/host_api_impl.cpp index 9974267fb2..4e4633dc48 100644 --- a/core/host_api/impl/host_api_impl.cpp +++ b/core/host_api/impl/host_api_impl.cpp @@ -14,7 +14,7 @@ #include "crypto/sr25519/sr25519_provider_impl.hpp" #include "host_api/impl/offchain_extension.hpp" #include "runtime/trie_storage_provider.hpp" -#include "host_api/impl/wasmedge/reg.h" +#include "runtime/wasmedge/register_host_api.hpp" namespace kagome::host_api { @@ -311,7 +311,7 @@ namespace kagome::host_api { // --------------------------- Offchain extension ---------------------------- - runtime::WasmI8 HostApiImpl::ext_offchain_is_validator_version_1() { + runtime::WasmI32 HostApiImpl::ext_offchain_is_validator_version_1() { return offchain_ext_.ext_offchain_is_validator_version_1(); } diff --git a/core/host_api/impl/host_api_impl.hpp b/core/host_api/impl/host_api_impl.hpp index 6949893b53..a24f950e39 100644 --- a/core/host_api/impl/host_api_impl.hpp +++ b/core/host_api/impl/host_api_impl.hpp @@ -57,34 +57,6 @@ namespace kagome::host_api { void reset() override; - void ext_offchain_index_set_version_1(I64, I64) {} - I32 ext_offchain_is_validator_version_1() { - return 0; - } - void ext_offchain_local_storage_clear_version_1(I32, I64) {} - I32 ext_offchain_local_storage_compare_and_set_version_1(I32, - I64, - I64, - I64) { - return 0; - } - I64 ext_offchain_local_storage_get_version_1(I32, I64) { - return 0; - } - void ext_offchain_local_storage_set_version_1(I32, I64, I64) {} - I64 ext_offchain_network_state_version_1() { - return 0; - } - I32 ext_offchain_random_seed_version_1() { - return 0; - } - I64 ext_offchain_submit_transaction_version_1(I64) { - return 0; - } - I64 ext_offchain_timestamp_version_1() { - return 0; - } - // ------------------------ Storage extensions v1 ------------------------ runtime::WasmSpan ext_storage_read_version_1( @@ -124,6 +96,18 @@ namespace kagome::host_api { void ext_storage_append_version_1(runtime::WasmSpan key, runtime::WasmSpan value) const override; + void ext_default_child_storage_clear_version_1(runtime::WasmSpan, + runtime::WasmSpan) const {} + runtime::WasmSpan ext_default_child_storage_next_key_version_1( + runtime::WasmSpan, runtime::WasmSpan) const { + return 0; + } + void ext_default_child_storage_set_version_1(runtime::WasmSpan, + runtime::WasmSpan, + runtime::WasmSpan) const {} + void ext_default_child_storage_storage_kill_version_1( + runtime::WasmSpan) const {} + runtime::WasmPointer ext_trie_blake2_256_root_version_1( runtime::WasmSpan values_data) override; @@ -232,7 +216,7 @@ namespace kagome::host_api { // -------------------------- Offchain extension --------------------------- - runtime::WasmI8 ext_offchain_is_validator_version_1() override; + runtime::WasmI32 ext_offchain_is_validator_version_1() override; runtime::WasmSpan ext_offchain_submit_transaction_version_1( runtime::WasmSpan data) override; diff --git a/core/log/configurator.cpp b/core/log/configurator.cpp index 13aa393cd5..39a644f37d 100644 --- a/core/log/configurator.cpp +++ b/core/log/configurator.cpp @@ -52,10 +52,10 @@ namespace kagome::log { - name: block_executor - name: block_validator - name: grandpa + level: trace children: - name: voting_round - name: runtime - level: trace children: - name: runtime_api - name: host_api diff --git a/core/runtime/wasmedge/core_api_factory_impl.cpp b/core/runtime/wasmedge/core_api_factory_impl.cpp index b31de750b5..2fb288ed29 100644 --- a/core/runtime/wasmedge/core_api_factory_impl.cpp +++ b/core/runtime/wasmedge/core_api_factory_impl.cpp @@ -27,7 +27,8 @@ namespace kagome::runtime::wasmedge { outcome::result> getInstanceAt( std::shared_ptr, - const primitives::BlockInfo &b) override { + const primitives::BlockInfo &, + const primitives::BlockHeader &) override { if (instance_ == nullptr) { OUTCOME_TRY(module, ModuleImpl::createFromCode(code_, env_factory_)); OUTCOME_TRY(inst, module->instantiate()); diff --git a/core/runtime/wasmedge/core_api_factory_impl.hpp b/core/runtime/wasmedge/core_api_factory_impl.hpp index 40908329aa..e21c848d8b 100644 --- a/core/runtime/wasmedge/core_api_factory_impl.hpp +++ b/core/runtime/wasmedge/core_api_factory_impl.hpp @@ -6,7 +6,7 @@ #ifndef KAGOME_CORE_RUNTIME_WASMEDGE_CORE_API_FACTORY_IMPL_HPP #define KAGOME_CORE_RUNTIME_WASMEDGE_CORE_API_FACTORY_IMPL_HPP -#include "runtime/core_api_factory_impl.hpp" +#include "runtime/core_api_factory.hpp" #include "runtime/runtime_api/core.hpp" namespace kagome::storage::changes_trie { diff --git a/core/runtime/wasmedge/memory_impl.cpp b/core/runtime/wasmedge/memory_impl.cpp index 6c479e955f..980ffb7ae4 100644 --- a/core/runtime/wasmedge/memory_impl.cpp +++ b/core/runtime/wasmedge/memory_impl.cpp @@ -57,7 +57,7 @@ namespace kagome::runtime::wasmedge { return allocator_->allocate(size); } - boost::optional MemoryImpl::deallocate(WasmPointer ptr) { + std::optional MemoryImpl::deallocate(WasmPointer ptr) { return allocator_->deallocate(ptr); } diff --git a/core/runtime/wasmedge/memory_impl.hpp b/core/runtime/wasmedge/memory_impl.hpp index 781a049909..4f00373082 100644 --- a/core/runtime/wasmedge/memory_impl.hpp +++ b/core/runtime/wasmedge/memory_impl.hpp @@ -38,7 +38,7 @@ namespace kagome::runtime::wasmedge { ~MemoryImpl() override = default; WasmPointer allocate(WasmSize size) override; - boost::optional deallocate(WasmPointer ptr) override; + std::optional deallocate(WasmPointer ptr) override; int8_t load8s(WasmPointer addr) const override; uint8_t load8u(WasmPointer addr) const override; diff --git a/core/runtime/wasmedge/memory_provider.cpp b/core/runtime/wasmedge/memory_provider.cpp index c48964c162..90676f3fa9 100644 --- a/core/runtime/wasmedge/memory_provider.cpp +++ b/core/runtime/wasmedge/memory_provider.cpp @@ -9,13 +9,15 @@ #include "runtime/wasmedge/memory_impl.hpp" #include +#include namespace kagome::runtime::wasmedge { - boost::optional WasmedgeMemoryProvider::getCurrentMemory() - const { - return memory_ == nullptr ? boost::none - : boost::optional{*memory_}; + std::optional> + WasmedgeMemoryProvider::getCurrentMemory() const { + return memory_ == nullptr + ? std::nullopt + : std::optional>{*memory_}; } outcome::result WasmedgeMemoryProvider::resetMemory(WasmSize) { diff --git a/core/runtime/wasmedge/memory_provider.hpp b/core/runtime/wasmedge/memory_provider.hpp index 7a0ff38836..75b1042c0e 100644 --- a/core/runtime/wasmedge/memory_provider.hpp +++ b/core/runtime/wasmedge/memory_provider.hpp @@ -17,7 +17,8 @@ namespace kagome::runtime::wasmedge { class WasmedgeMemoryProvider final : public MemoryProvider { public: - boost::optional getCurrentMemory() const override; + std::optional> getCurrentMemory() + const override; [[nodiscard]] outcome::result resetMemory( WasmSize heap_base) override; @@ -26,7 +27,7 @@ namespace kagome::runtime::wasmedge { private: std::shared_ptr memory_; - WasmEdge_MemoryInstanceContext* mem_ctx_; + WasmEdge_MemoryInstanceContext *mem_ctx_; }; } // namespace kagome::runtime::wasmedge diff --git a/core/runtime/wasmedge/module_impl.cpp b/core/runtime/wasmedge/module_impl.cpp index 27caf194ca..9f3c9f05bd 100644 --- a/core/runtime/wasmedge/module_impl.cpp +++ b/core/runtime/wasmedge/module_impl.cpp @@ -5,12 +5,14 @@ #include "runtime/wasmedge/module_impl.hpp" +#include #include #include #include #include "common/mp_utils.hpp" +#include "crypto/sha/sha256.hpp" #include "runtime/wasmedge/instance_environment_factory.hpp" #include "runtime/wasmedge/memory_provider.hpp" #include "runtime/wasmedge/module_instance_impl.hpp" @@ -41,17 +43,24 @@ namespace kagome::runtime::wasmedge { ConfCtx, WasmEdge_CompilerOptimizationLevel_O3); auto CompilerCtx = WasmEdge_CompilerCreate(ConfCtx); - std::ofstream ofs; - ofs.open("source.wasm"); - ofs.write(reinterpret_cast(code.data()), code.size()); - ofs.close(); + auto hash = crypto::sha256(code); + std::string source = "source-" + hash.toHex() + ".wasm"; + std::string result = "result-" + hash.toHex() + ".wasm.so"; - WasmEdge_CompilerCompile(CompilerCtx, "source.wasm", "result.wasm.so"); + if (not std::filesystem::exists(source)) { + std::ofstream ofs; + ofs.open(source); + ofs.write(reinterpret_cast(code.data()), code.size()); + ofs.close(); + } + + if (not std::filesystem::exists(result)) { + WasmEdge_CompilerCompile(CompilerCtx, source.c_str(), result.c_str()); + } auto *LoaderCtx = WasmEdge_LoaderCreate(ConfCtx); WasmEdge_ASTModuleContext *ASTCtx = nullptr; - auto Res = - WasmEdge_LoaderParseFromFile(LoaderCtx, &ASTCtx, "result.wasm.so"); + auto Res = WasmEdge_LoaderParseFromFile(LoaderCtx, &ASTCtx, result.c_str()); std::unique_ptr wasm_module_impl( new ModuleImpl(ASTCtx, std::move(env_factory))); diff --git a/core/runtime/wasmedge/module_instance_impl.cpp b/core/runtime/wasmedge/module_instance_impl.cpp index 6da349467d..571dbe6620 100644 --- a/core/runtime/wasmedge/module_instance_impl.cpp +++ b/core/runtime/wasmedge/module_instance_impl.cpp @@ -21,15 +21,16 @@ namespace kagome::runtime::wasmedge { logger_{log::createLogger("ModuleInstance", "wasmedge")} { rei_ = WasmEdge_VMCreate(NULL, NULL); WasmEdge_String ModuleName = WasmEdge_StringCreateByCString("env"); -// auto Res = WasmEdge_VMRegisterModuleFromASTModule( -// rei_, ModuleName, parent_->ast()); + // auto Res = WasmEdge_VMRegisterModuleFromASTModule( + // rei_, ModuleName, parent_->ast()); WasmEdge_ImportObjectContext *ImpObj = WasmEdge_ImportObjectCreate(ModuleName, env_.host_api.get()); -// WasmEdge_ImportObjectContext *ImpObj = -// WasmEdge_VMGetImportModuleContext(rei_, WasmEdge_HostRegistration_WasmEdge_Process); + // WasmEdge_ImportObjectContext *ImpObj = + // WasmEdge_VMGetImportModuleContext(rei_, + // WasmEdge_HostRegistration_WasmEdge_Process); WasmEdge_StringDelete(ModuleName); register_host_api(ImpObj); - dynamic_cast(env_.memory_provider.get()) + dynamic_cast(env_.memory_provider.get()) ->setExternalInterface(ImpObj); auto Res = WasmEdge_VMRegisterModuleFromImport(rei_, ImpObj); // interpreter_ = WasmEdge_InterpreterCreate(nullptr, nullptr); @@ -53,7 +54,7 @@ namespace kagome::runtime::wasmedge { return PtrSize{i}; } - outcome::result> ModuleInstanceImpl::getGlobal( + outcome::result> ModuleInstanceImpl::getGlobal( std::string_view name) const { auto GlobalName = WasmEdge_StringCreateByCString(name.data()); auto store = WasmEdge_VMGetStoreContext(rei_); @@ -71,7 +72,7 @@ namespace kagome::runtime::wasmedge { case WasmEdge_ValType_F64: return WasmValue{WasmEdge_ValueGetF64(val)}; default: - return boost::none; + return std::nullopt; } } diff --git a/core/runtime/wasmedge/module_instance_impl.hpp b/core/runtime/wasmedge/module_instance_impl.hpp index 49d1dcd2de..6df6b64337 100644 --- a/core/runtime/wasmedge/module_instance_impl.hpp +++ b/core/runtime/wasmedge/module_instance_impl.hpp @@ -23,7 +23,7 @@ namespace kagome::runtime::wasmedge { outcome::result callExportFunction(std::string_view name, PtrSize args) const override; - outcome::result> getGlobal( + outcome::result> getGlobal( std::string_view name) const override; InstanceEnvironment const &getEnvironment() const override; diff --git a/core/runtime/wasmedge/register_host_api.hpp b/core/runtime/wasmedge/register_host_api.hpp index b7d36258e3..8dfd96bdc3 100644 --- a/core/runtime/wasmedge/register_host_api.hpp +++ b/core/runtime/wasmedge/register_host_api.hpp @@ -250,8 +250,12 @@ inline void register_host_api(WasmEdge_ImportObjectContext *ImpObj) { HostApiImpl, ext_crypto_finish_batch_verify_version_1, ImpObj); REGISTER_HOST_API_FUNC( HostApiImpl, ext_crypto_secp256k1_ecdsa_recover_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_crypto_secp256k1_ecdsa_recover_version_2, ImpObj); REGISTER_HOST_API_FUNC( HostApi, ext_crypto_secp256k1_ecdsa_recover_compressed_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApi, ext_crypto_secp256k1_ecdsa_recover_compressed_version_2, ImpObj); REGISTER_HOST_API_FUNC( HostApiImpl, ext_crypto_sr25519_generate_version_1, ImpObj); REGISTER_HOST_API_FUNC( @@ -312,4 +316,14 @@ inline void register_host_api(WasmEdge_ImportObjectContext *ImpObj) { REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_set_version_1, ImpObj); REGISTER_HOST_API_FUNC( HostApiImpl, ext_storage_start_transaction_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_default_child_storage_clear_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_default_child_storage_get_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_default_child_storage_next_key_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_default_child_storage_set_version_1, ImpObj); + REGISTER_HOST_API_FUNC( + HostApiImpl, ext_default_child_storage_storage_kill_version_1, ImpObj); } diff --git a/core/utils/profiler.hpp b/core/utils/profiler.hpp index a0b76b055c..12c333c643 100644 --- a/core/utils/profiler.hpp +++ b/core/utils/profiler.hpp @@ -1,15 +1,36 @@ #pragma once #include "log/logger.hpp" +#include "log/profiling_logger.hpp" #include +namespace { + + template + constexpr std::string unit() { + if constexpr (std::is_same_v) { + return "sec"; + } else if constexpr (std::is_same_v) { + return "ms"; + } + return ""; + } + +} // namespace + +template class TicToc { std::string name_; const kagome::log::Logger &log_; std::chrono::time_point t_; public: + TicToc() : TicToc("") {} + + TicToc(const std::string &name) + : TicToc(name, ::kagome::log::profiling_logger) {} + TicToc(const std::string &name, const kagome::log::Logger &log) : name_(name), log_(log) { t_ = std::chrono::high_resolution_clock::now(); @@ -22,10 +43,10 @@ class TicToc { if (line != -1) { str += "at line " + std::to_string(line); } - log_->info( - "{} lasted for {} sec", - str, - std::chrono::duration_cast(t_ - prev).count()); + log_->warn("{} lasted for {} {}", + str, + std::chrono::duration_cast(t_ - prev).count(), + unit()); } ~TicToc() { From 3a344c0fba457b98800c307d1e6ebb54490a83c5 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Fri, 24 Dec 2021 23:28:54 +0300 Subject: [PATCH 13/43] fix2 --- core/host_api/host_api.hpp | 22 ++++++------- core/host_api/impl/host_api_impl.cpp | 22 ++++++------- core/host_api/impl/host_api_impl.hpp | 31 +++++++------------ core/host_api/impl/offchain_extension.cpp | 22 ++++++------- core/host_api/impl/offchain_extension.hpp | 22 ++++++------- core/runtime/types.hpp | 3 +- .../wasmedge/instance_environment_factory.cpp | 13 +++++--- .../wasmedge/instance_environment_factory.hpp | 4 +++ test/mock/core/host_api/host_api_mock.hpp | 22 ++++++------- 9 files changed, 79 insertions(+), 82 deletions(-) diff --git a/core/host_api/host_api.hpp b/core/host_api/host_api.hpp index 0a2e945780..cc81217439 100644 --- a/core/host_api/host_api.hpp +++ b/core/host_api/host_api.hpp @@ -370,7 +370,7 @@ namespace kagome::host_api { // -------------------------- Offchain extension --------------------------- /// @copydoc OffchainExtension::ext_offchain_is_validator_version_1 - [[nodiscard]] virtual runtime::WasmI32 + [[nodiscard]] virtual runtime::WasmU32 ext_offchain_is_validator_version_1() = 0; /// @copydoc OffchainExtension::ext_offchain_submit_transaction_version_1 @@ -395,26 +395,26 @@ namespace kagome::host_api { /// @copydoc OffchainExtension::ext_offchain_local_storage_set_version_1 virtual void ext_offchain_local_storage_set_version_1( - runtime::WasmI32 kind, + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan value) = 0; /// @copydoc OffchainExtension::ext_offchain_local_storage_clear_version_1 virtual void ext_offchain_local_storage_clear_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key) = 0; + runtime::WasmU32 kind, runtime::WasmSpan key) = 0; /// @copydoc /// OffchainExtension::ext_offchain_local_storage_compare_and_set_version_1 - [[nodiscard]] virtual runtime::WasmI8 + [[nodiscard]] virtual runtime::WasmU32 ext_offchain_local_storage_compare_and_set_version_1( - runtime::WasmI32 kind, + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan expected, runtime::WasmSpan value) = 0; /// @copydoc OffchainExtension::ext_offchain_local_storage_get_version_1 [[nodiscard]] virtual runtime::WasmSpan - ext_offchain_local_storage_get_version_1(runtime::WasmI32 kind, + ext_offchain_local_storage_get_version_1(runtime::WasmU32 kind, runtime::WasmSpan key) = 0; /// @copydoc OffchainExtension::ext_offchain_http_request_start_version_1 @@ -426,7 +426,7 @@ namespace kagome::host_api { /// @copydoc /// OffchainExtension::ext_offchain_http_request_add_header_version_1 [[nodiscard]] virtual runtime::WasmSpan - ext_offchain_http_request_add_header_version_1(runtime::WasmI32 request_id, + ext_offchain_http_request_add_header_version_1(runtime::WasmU32 request_id, runtime::WasmSpan name, runtime::WasmSpan value) = 0; @@ -434,7 +434,7 @@ namespace kagome::host_api { /// OffchainExtension::ext_offchain_http_request_write_body_version_1 [[nodiscard]] virtual runtime::WasmSpan ext_offchain_http_request_write_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan chunk, runtime::WasmSpan deadline) = 0; @@ -445,19 +445,19 @@ namespace kagome::host_api { /// @copydoc OffchainExtension::ext_offchain_http_response_headers_version_1 virtual runtime::WasmSpan ext_offchain_http_response_headers_version_1( - runtime::WasmI32 request_id) = 0; + runtime::WasmU32 request_id) = 0; /// @copydoc /// OffchainExtension::ext_offchain_http_response_read_body_version_1 [[nodiscard]] virtual runtime::WasmSpan ext_offchain_http_response_read_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan buffer, runtime::WasmSpan deadline) = 0; /// @copydoc OffchainExtension::ext_offchain_set_authorized_nodes_version_1 virtual void ext_offchain_set_authorized_nodes_version_1( - runtime::WasmSpan nodes, runtime::WasmI32 authorized_only) = 0; + runtime::WasmSpan nodes, runtime::WasmU32 authorized_only) = 0; /// @copydoc OffchainExtension::ext_offchain_index_set_version_1 virtual void ext_offchain_index_set_version_1(runtime::WasmSpan key, diff --git a/core/host_api/impl/host_api_impl.cpp b/core/host_api/impl/host_api_impl.cpp index 4e4633dc48..a5c51705d0 100644 --- a/core/host_api/impl/host_api_impl.cpp +++ b/core/host_api/impl/host_api_impl.cpp @@ -311,7 +311,7 @@ namespace kagome::host_api { // --------------------------- Offchain extension ---------------------------- - runtime::WasmI32 HostApiImpl::ext_offchain_is_validator_version_1() { + runtime::WasmU32 HostApiImpl::ext_offchain_is_validator_version_1() { return offchain_ext_.ext_offchain_is_validator_version_1(); } @@ -338,19 +338,19 @@ namespace kagome::host_api { } void HostApiImpl::ext_offchain_local_storage_set_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key, runtime::WasmSpan value) { + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan value) { return offchain_ext_.ext_offchain_local_storage_set_version_1( kind, key, value); } void HostApiImpl::ext_offchain_local_storage_clear_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key) { + runtime::WasmU32 kind, runtime::WasmSpan key) { return offchain_ext_.ext_offchain_local_storage_clear_version_1(kind, key); } - runtime::WasmI8 + runtime::WasmU32 HostApiImpl::ext_offchain_local_storage_compare_and_set_version_1( - runtime::WasmI32 kind, + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan expected, runtime::WasmSpan value) { @@ -359,7 +359,7 @@ namespace kagome::host_api { } runtime::WasmSpan HostApiImpl::ext_offchain_local_storage_get_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key) { + runtime::WasmU32 kind, runtime::WasmSpan key) { return offchain_ext_.ext_offchain_local_storage_get_version_1(kind, key); } @@ -370,7 +370,7 @@ namespace kagome::host_api { } runtime::WasmSpan HostApiImpl::ext_offchain_http_request_add_header_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan name, runtime::WasmSpan value) { return offchain_ext_.ext_offchain_http_request_add_header_version_1( @@ -378,7 +378,7 @@ namespace kagome::host_api { } runtime::WasmSpan HostApiImpl::ext_offchain_http_request_write_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan chunk, runtime::WasmSpan deadline) { return offchain_ext_.ext_offchain_http_request_write_body_version_1( @@ -392,13 +392,13 @@ namespace kagome::host_api { } runtime::WasmSpan HostApiImpl::ext_offchain_http_response_headers_version_1( - runtime::WasmI32 request_id) { + runtime::WasmU32 request_id) { return offchain_ext_.ext_offchain_http_response_headers_version_1( request_id); } runtime::WasmSpan HostApiImpl::ext_offchain_http_response_read_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan buffer, runtime::WasmSpan deadline) { return offchain_ext_.ext_offchain_http_response_read_body_version_1( @@ -406,7 +406,7 @@ namespace kagome::host_api { } void HostApiImpl::ext_offchain_set_authorized_nodes_version_1( - runtime::WasmSpan nodes, runtime::WasmI32 authorized_only) { + runtime::WasmSpan nodes, runtime::WasmU32 authorized_only) { return offchain_ext_.ext_offchain_set_authorized_nodes_version_1( nodes, authorized_only); } diff --git a/core/host_api/impl/host_api_impl.hpp b/core/host_api/impl/host_api_impl.hpp index a24f950e39..900605f9c8 100644 --- a/core/host_api/impl/host_api_impl.hpp +++ b/core/host_api/impl/host_api_impl.hpp @@ -96,15 +96,6 @@ namespace kagome::host_api { void ext_storage_append_version_1(runtime::WasmSpan key, runtime::WasmSpan value) const override; - void ext_default_child_storage_clear_version_1(runtime::WasmSpan, - runtime::WasmSpan) const {} - runtime::WasmSpan ext_default_child_storage_next_key_version_1( - runtime::WasmSpan, runtime::WasmSpan) const { - return 0; - } - void ext_default_child_storage_set_version_1(runtime::WasmSpan, - runtime::WasmSpan, - runtime::WasmSpan) const {} void ext_default_child_storage_storage_kill_version_1( runtime::WasmSpan) const {} @@ -216,7 +207,7 @@ namespace kagome::host_api { // -------------------------- Offchain extension --------------------------- - runtime::WasmI32 ext_offchain_is_validator_version_1() override; + runtime::WasmU32 ext_offchain_is_validator_version_1() override; runtime::WasmSpan ext_offchain_submit_transaction_version_1( runtime::WasmSpan data) override; @@ -230,21 +221,21 @@ namespace kagome::host_api { runtime::WasmPointer ext_offchain_random_seed_version_1() override; void ext_offchain_local_storage_set_version_1( - runtime::WasmI32 kind, + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan value) override; void ext_offchain_local_storage_clear_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key) override; + runtime::WasmU32 kind, runtime::WasmSpan key) override; - runtime::WasmI8 ext_offchain_local_storage_compare_and_set_version_1( - runtime::WasmI32 kind, + runtime::WasmU32 ext_offchain_local_storage_compare_and_set_version_1( + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan expected, runtime::WasmSpan value) override; runtime::WasmSpan ext_offchain_local_storage_get_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key) override; + runtime::WasmU32 kind, runtime::WasmSpan key) override; runtime::WasmSpan ext_offchain_http_request_start_version_1( runtime::WasmSpan method, @@ -252,12 +243,12 @@ namespace kagome::host_api { runtime::WasmSpan meta) override; runtime::WasmSpan ext_offchain_http_request_add_header_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan name, runtime::WasmSpan value) override; runtime::WasmSpan ext_offchain_http_request_write_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan chunk, runtime::WasmSpan deadline) override; @@ -265,15 +256,15 @@ namespace kagome::host_api { runtime::WasmSpan ids, runtime::WasmSpan deadline) override; runtime::WasmSpan ext_offchain_http_response_headers_version_1( - runtime::WasmI32 request_id) override; + runtime::WasmU32 request_id) override; runtime::WasmSpan ext_offchain_http_response_read_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan buffer, runtime::WasmSpan deadline) override; void ext_offchain_set_authorized_nodes_version_1( - runtime::WasmSpan nodes, runtime::WasmI32 authorized_only) override; + runtime::WasmSpan nodes, runtime::WasmU32 authorized_only) override; void ext_offchain_index_set_version_1(runtime::WasmSpan key, runtime::WasmSpan value) override; diff --git a/core/host_api/impl/offchain_extension.cpp b/core/host_api/impl/offchain_extension.cpp index 140230ac75..c9f6c0ce33 100644 --- a/core/host_api/impl/offchain_extension.cpp +++ b/core/host_api/impl/offchain_extension.cpp @@ -42,7 +42,7 @@ namespace kagome::host_api { return worker_opt.value(); } - runtime::WasmI8 OffchainExtension::ext_offchain_is_validator_version_1() { + runtime::WasmU32 OffchainExtension::ext_offchain_is_validator_version_1() { auto &worker = getWorker(); bool isValidator = worker.isValidator(); return isValidator ? 1 : 0; @@ -96,7 +96,7 @@ namespace kagome::host_api { } void OffchainExtension::ext_offchain_local_storage_set_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key, runtime::WasmSpan value) { + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan value) { auto &worker = getWorker(); auto &memory = memory_provider_->getCurrentMemory()->get(); @@ -123,7 +123,7 @@ namespace kagome::host_api { } void OffchainExtension::ext_offchain_local_storage_clear_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key) { + runtime::WasmU32 kind, runtime::WasmSpan key) { auto &worker = getWorker(); auto &memory = memory_provider_->getCurrentMemory()->get(); @@ -147,9 +147,9 @@ namespace kagome::host_api { worker.localStorageClear(storage_type, key_buffer); } - runtime::WasmI8 + runtime::WasmU32 OffchainExtension::ext_offchain_local_storage_compare_and_set_version_1( - runtime::WasmI32 kind, + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan expected, runtime::WasmSpan value) { @@ -184,7 +184,7 @@ namespace kagome::host_api { } runtime::WasmSpan OffchainExtension::ext_offchain_local_storage_get_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key) { + runtime::WasmU32 kind, runtime::WasmSpan key) { auto &worker = getWorker(); auto &memory = memory_provider_->getCurrentMemory()->get(); @@ -267,7 +267,7 @@ namespace kagome::host_api { runtime::WasmSpan OffchainExtension::ext_offchain_http_request_add_header_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan name_pos, runtime::WasmSpan value_pos) { auto &worker = getWorker(); @@ -302,7 +302,7 @@ namespace kagome::host_api { runtime::WasmSpan OffchainExtension::ext_offchain_http_request_write_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan chunk_pos, runtime::WasmSpan deadline_pos) { auto &worker = getWorker(); @@ -358,7 +358,7 @@ namespace kagome::host_api { runtime::WasmSpan OffchainExtension::ext_offchain_http_response_headers_version_1( - runtime::WasmI32 request_id) { + runtime::WasmU32 request_id) { auto &worker = getWorker(); auto &memory = memory_provider_->getCurrentMemory()->get(); @@ -373,7 +373,7 @@ namespace kagome::host_api { runtime::WasmSpan OffchainExtension::ext_offchain_http_response_read_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan buffer_pos, runtime::WasmSpan deadline_pos) { auto &worker = getWorker(); @@ -403,7 +403,7 @@ namespace kagome::host_api { } void OffchainExtension::ext_offchain_set_authorized_nodes_version_1( - runtime::WasmSpan nodes_pos, runtime::WasmI32 authorized_only) { + runtime::WasmSpan nodes_pos, runtime::WasmU32 authorized_only) { auto &worker = getWorker(); auto &memory = memory_provider_->getCurrentMemory()->get(); diff --git a/core/host_api/impl/offchain_extension.hpp b/core/host_api/impl/offchain_extension.hpp index c4fec4e2b7..7488e82176 100644 --- a/core/host_api/impl/offchain_extension.hpp +++ b/core/host_api/impl/offchain_extension.hpp @@ -43,7 +43,7 @@ namespace kagome::host_api { * @return a boolean equal to true if the node is a validator, false if * otherwise */ - runtime::WasmI8 ext_offchain_is_validator_version_1(); + runtime::WasmU32 ext_offchain_is_validator_version_1(); /** * @brief Given a SCALE encoded extrinsic, this function submits the @@ -121,7 +121,7 @@ namespace kagome::host_api { * @param key a pointer-size indicating the key * @param value a pointer-size indicating the value */ - void ext_offchain_local_storage_set_version_1(runtime::WasmI32 kind, + void ext_offchain_local_storage_set_version_1(runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan value); @@ -138,7 +138,7 @@ namespace kagome::host_api { * storage * @param key a pointer-size indicating the key */ - void ext_offchain_local_storage_clear_version_1(runtime::WasmI32 kind, + void ext_offchain_local_storage_clear_version_1(runtime::WasmU32 kind, runtime::WasmSpan key); /** @@ -162,8 +162,8 @@ namespace kagome::host_api { * @return a boolean equal to true if the new value has been set, false if * otherwise */ - runtime::WasmI8 ext_offchain_local_storage_compare_and_set_version_1( - runtime::WasmI32 kind, + runtime::WasmU32 ext_offchain_local_storage_compare_and_set_version_1( + runtime::WasmU32 kind, runtime::WasmSpan key, runtime::WasmSpan expected, runtime::WasmSpan value); @@ -185,7 +185,7 @@ namespace kagome::host_api { * the value or the corresponding key */ runtime::WasmSpan ext_offchain_local_storage_get_version_1( - runtime::WasmI32 kind, runtime::WasmSpan key); + runtime::WasmU32 kind, runtime::WasmSpan key); /** * @brief Initiates a HTTP request given by the HTTP method and the URL. @@ -233,7 +233,7 @@ namespace kagome::host_api { * failure is implementation specific */ runtime::WasmSpan ext_offchain_http_request_add_header_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan name, runtime::WasmSpan value); @@ -258,7 +258,7 @@ namespace kagome::host_api { * type */ runtime::WasmSpan ext_offchain_http_request_write_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan chunk, runtime::WasmSpan deadline); @@ -298,7 +298,7 @@ namespace kagome::host_api { * pairs */ runtime::WasmSpan ext_offchain_http_response_headers_version_1( - runtime::WasmI32 request_id); + runtime::WasmU32 request_id); /** * @brief Reads a chunk of body response to the given buffer. Returns the @@ -322,7 +322,7 @@ namespace kagome::host_api { * written or a `HTTP error` type on failure */ runtime::WasmSpan ext_offchain_http_response_read_body_version_1( - runtime::WasmI32 request_id, + runtime::WasmU32 request_id, runtime::WasmSpan buffer, runtime::WasmSpan deadline); @@ -344,7 +344,7 @@ namespace kagome::host_api { * rejected. If set to 0, then no such restriction is placed */ void ext_offchain_set_authorized_nodes_version_1( - runtime::WasmSpan nodes, runtime::WasmI32 authorized_only); + runtime::WasmSpan nodes, runtime::WasmU32 authorized_only); /** * @brief Write a key value pair to the offchain database in a buffered diff --git a/core/runtime/types.hpp b/core/runtime/types.hpp index 31746be06a..5cf0d1742f 100644 --- a/core/runtime/types.hpp +++ b/core/runtime/types.hpp @@ -43,8 +43,7 @@ namespace kagome::runtime { */ using WasmOffset = uint32_t; - using WasmI8 = int8_t; - using WasmI32 = int32_t; + using WasmU32 = uint32_t; using WasmU64 = uint64_t; /** diff --git a/core/runtime/wasmedge/instance_environment_factory.cpp b/core/runtime/wasmedge/instance_environment_factory.cpp index bf01752e12..8b25dae1d5 100644 --- a/core/runtime/wasmedge/instance_environment_factory.cpp +++ b/core/runtime/wasmedge/instance_environment_factory.cpp @@ -16,22 +16,25 @@ namespace kagome::runtime::wasmedge { InstanceEnvironmentFactory::InstanceEnvironmentFactory( std::shared_ptr storage, + std::shared_ptr serializer, std::shared_ptr host_api_factory, std::shared_ptr block_header_repo, std::shared_ptr changes_tracker) : storage_{std::move(storage)}, + serializer_{std::move(serializer)}, host_api_factory_{std::move(host_api_factory)}, block_header_repo_{std::move(block_header_repo)}, changes_tracker_{std::move(changes_tracker)} { - BOOST_ASSERT(storage_); - BOOST_ASSERT(host_api_factory_); - BOOST_ASSERT(block_header_repo_); - BOOST_ASSERT(changes_tracker_); + BOOST_ASSERT(storage_ != nullptr); + BOOST_ASSERT(serializer_ != nullptr); + BOOST_ASSERT(host_api_factory_ != nullptr); + BOOST_ASSERT(block_header_repo_ != nullptr); + BOOST_ASSERT(changes_tracker_ != nullptr); } WasmedgeInstanceEnvironment InstanceEnvironmentFactory::make() const { auto new_storage_provider = - std::make_shared(storage_); + std::make_shared(storage_, serializer_); auto core_factory = std::make_shared( shared_from_this(), block_header_repo_, changes_tracker_); auto memory_provider = std::make_shared(); diff --git a/core/runtime/wasmedge/instance_environment_factory.hpp b/core/runtime/wasmedge/instance_environment_factory.hpp index 05dd2e6368..a8b0114a20 100644 --- a/core/runtime/wasmedge/instance_environment_factory.hpp +++ b/core/runtime/wasmedge/instance_environment_factory.hpp @@ -7,9 +7,11 @@ #define KAGOME_CORE_RUNTIME_WASMEDGE_INSTANCE_ENVIRONMENT_FACTORY_HPP #include "runtime/instance_environment.hpp" +#include "storage/trie/serialization/trie_serializer.hpp" namespace kagome::storage::trie { class TrieStorage; + class TrieSerializer; } namespace kagome::storage::changes_trie { @@ -39,6 +41,7 @@ namespace kagome::runtime::wasmedge { public: InstanceEnvironmentFactory( std::shared_ptr storage, + std::shared_ptr serializer, std::shared_ptr host_api_factory, std::shared_ptr block_header_repo, std::shared_ptr changes_tracker); @@ -47,6 +50,7 @@ namespace kagome::runtime::wasmedge { private: std::shared_ptr storage_; + std::shared_ptr serializer_; std::shared_ptr host_api_factory_; std::shared_ptr block_header_repo_; std::shared_ptr changes_tracker_; diff --git a/test/mock/core/host_api/host_api_mock.hpp b/test/mock/core/host_api/host_api_mock.hpp index c5a4bc655e..9385056926 100644 --- a/test/mock/core/host_api/host_api_mock.hpp +++ b/test/mock/core/host_api/host_api_mock.hpp @@ -260,7 +260,7 @@ namespace kagome::host_api { // -------------------------- Offchain extension --------------------------- - MOCK_METHOD(runtime::WasmI8, + MOCK_METHOD(runtime::WasmU32, ext_offchain_is_validator_version_1, (), (override)); @@ -292,17 +292,17 @@ namespace kagome::host_api { MOCK_METHOD(void, ext_offchain_local_storage_set_version_1, - (runtime::WasmI32, runtime::WasmSpan, runtime::WasmSpan), + (runtime::WasmU32, runtime::WasmSpan, runtime::WasmSpan), (override)); MOCK_METHOD(void, ext_offchain_local_storage_clear_version_1, - (runtime::WasmI32, runtime::WasmSpan), + (runtime::WasmU32, runtime::WasmSpan), (override)); - MOCK_METHOD(runtime::WasmI8, + MOCK_METHOD(runtime::WasmU32, ext_offchain_local_storage_compare_and_set_version_1, - (runtime::WasmI32, + (runtime::WasmU32, runtime::WasmSpan, runtime::WasmSpan, runtime::WasmSpan), @@ -310,7 +310,7 @@ namespace kagome::host_api { MOCK_METHOD(runtime::WasmSpan, ext_offchain_local_storage_get_version_1, - (runtime::WasmI32, runtime::WasmSpan), + (runtime::WasmU32, runtime::WasmSpan), (override)); MOCK_METHOD(runtime::WasmSpan, @@ -320,12 +320,12 @@ namespace kagome::host_api { MOCK_METHOD(runtime::WasmSpan, ext_offchain_http_request_add_header_version_1, - (runtime::WasmI32, runtime::WasmSpan, runtime::WasmSpan), + (runtime::WasmU32, runtime::WasmSpan, runtime::WasmSpan), (override)); MOCK_METHOD(runtime::WasmSpan, ext_offchain_http_request_write_body_version_1, - (runtime::WasmI32, runtime::WasmSpan, runtime::WasmSpan), + (runtime::WasmU32, runtime::WasmSpan, runtime::WasmSpan), (override)); MOCK_METHOD(runtime::WasmSpan, @@ -335,17 +335,17 @@ namespace kagome::host_api { MOCK_METHOD(runtime::WasmSpan, ext_offchain_http_response_headers_version_1, - (runtime::WasmI32), + (runtime::WasmU32), (override)); MOCK_METHOD(runtime::WasmSpan, ext_offchain_http_response_read_body_version_1, - (runtime::WasmI32, runtime::WasmSpan, runtime::WasmSpan), + (runtime::WasmU32, runtime::WasmSpan, runtime::WasmSpan), (override)); MOCK_METHOD(void, ext_offchain_set_authorized_nodes_version_1, - (runtime::WasmSpan, runtime::WasmI32), + (runtime::WasmSpan, runtime::WasmU32), (override)); MOCK_METHOD(void, From a56ba2d6606c5fcb0fbc6205b364f0cedde25b2c Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Wed, 5 Jan 2022 01:18:44 +0300 Subject: [PATCH 14/43] bump WasmEdge to 0.9.0 --- CMakeLists.txt | 7 +- cmake/conan.cmake | 14 + cmake/conan/conan.cmake | 909 ++++++++++++++++++ cmake/kagomeConfig.cmake.in | 6 + conanfile.txt | 11 - core/host_api/CMakeLists.txt | 2 +- core/host_api/impl/host_api_impl.cpp | 1 - core/log/configurator.cpp | 1 + core/runtime/memory.hpp | 2 +- core/runtime/wasmedge/CMakeLists.txt | 2 +- .../wasmedge/instance_environment_factory.cpp | 2 +- core/runtime/wasmedge/memory_impl.cpp | 2 +- core/runtime/wasmedge/memory_provider.cpp | 7 +- core/runtime/wasmedge/module_impl.cpp | 4 +- .../runtime/wasmedge/module_instance_impl.cpp | 11 +- core/runtime/wasmedge/register_host_api.hpp | 169 ++-- 16 files changed, 1019 insertions(+), 131 deletions(-) create mode 100644 cmake/conan.cmake create mode 100644 cmake/conan/conan.cmake delete mode 100644 conanfile.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f51c114c0c..360ed16c24 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,14 +19,15 @@ include("cmake/Hunter/init.cmake") project(kagome C CXX) -include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) -conan_define_targets() - +include(cmake/conan.cmake) include(cmake/print.cmake) print("C flags: ${CMAKE_C_FLAGS}") print("CXX flags: ${CMAKE_CXX_FLAGS}") print("Using CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CMAKE_BINARY_DIR}) +find_package(WasmEdge REQUIRED) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads) link_libraries(Threads::Threads) diff --git a/cmake/conan.cmake b/cmake/conan.cmake new file mode 100644 index 0000000000..0cdbb4796e --- /dev/null +++ b/cmake/conan.cmake @@ -0,0 +1,14 @@ +set(CONAN "conan" CACHE STRING "Conan executable") +set(CONAN_PROFILE "default" CACHE STRING "Conan profile") + +include(cmake/conan/conan.cmake) + +conan_cmake_run( + PROFILE ${CONAN_PROFILE} + SETTINGS build_type=${CMAKE_BUILD_TYPE} + REQUIRES WasmEdge/0.9.0@sanblch/stable + GENERATORS cmake cmake_find_package_multi + CONAN_COMMAND ${CONAN}) + +include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) +conan_define_targets() diff --git a/cmake/conan/conan.cmake b/cmake/conan/conan.cmake new file mode 100644 index 0000000000..208ce24855 --- /dev/null +++ b/cmake/conan/conan.cmake @@ -0,0 +1,909 @@ +# The MIT License (MIT) + +# Copyright (c) 2018 JFrog + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + + +# This file comes from: https://github.com/conan-io/cmake-conan. Please refer +# to this repository for issues and documentation. + +# Its purpose is to wrap and launch Conan C/C++ Package Manager when cmake is called. +# It will take CMake current settings (os, compiler, compiler version, architecture) +# and translate them to conan settings for installing and retrieving dependencies. + +# It is intended to facilitate developers building projects that have conan dependencies, +# but it is only necessary on the end-user side. It is not necessary to create conan +# packages, in fact it shouldn't be use for that. Check the project documentation. + +# version: 0.18.0-dev + +include(CMakeParseArguments) + +function(_get_msvc_ide_version result) + set(${result} "" PARENT_SCOPE) + if(NOT MSVC_VERSION VERSION_LESS 1400 AND MSVC_VERSION VERSION_LESS 1500) + set(${result} 8 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1500 AND MSVC_VERSION VERSION_LESS 1600) + set(${result} 9 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1600 AND MSVC_VERSION VERSION_LESS 1700) + set(${result} 10 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1700 AND MSVC_VERSION VERSION_LESS 1800) + set(${result} 11 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1800 AND MSVC_VERSION VERSION_LESS 1900) + set(${result} 12 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1900 AND MSVC_VERSION VERSION_LESS 1910) + set(${result} 14 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1910 AND MSVC_VERSION VERSION_LESS 1920) + set(${result} 15 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1920 AND MSVC_VERSION VERSION_LESS 1930) + set(${result} 16 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1930 AND MSVC_VERSION VERSION_LESS 1940) + set(${result} 17 PARENT_SCOPE) + else() + message(FATAL_ERROR "Conan: Unknown MSVC compiler version [${MSVC_VERSION}]") + endif() +endfunction() + +macro(_conan_detect_build_type) + conan_parse_arguments(${ARGV}) + + if(ARGUMENTS_BUILD_TYPE) + set(_CONAN_SETTING_BUILD_TYPE ${ARGUMENTS_BUILD_TYPE}) + elseif(CMAKE_BUILD_TYPE) + set(_CONAN_SETTING_BUILD_TYPE ${CMAKE_BUILD_TYPE}) + else() + message(FATAL_ERROR "Please specify in command line CMAKE_BUILD_TYPE (-DCMAKE_BUILD_TYPE=Release)") + endif() + + string(TOUPPER ${_CONAN_SETTING_BUILD_TYPE} _CONAN_SETTING_BUILD_TYPE_UPPER) + if (_CONAN_SETTING_BUILD_TYPE_UPPER STREQUAL "DEBUG") + set(_CONAN_SETTING_BUILD_TYPE "Debug") + elseif(_CONAN_SETTING_BUILD_TYPE_UPPER STREQUAL "RELEASE") + set(_CONAN_SETTING_BUILD_TYPE "Release") + elseif(_CONAN_SETTING_BUILD_TYPE_UPPER STREQUAL "RELWITHDEBINFO") + set(_CONAN_SETTING_BUILD_TYPE "RelWithDebInfo") + elseif(_CONAN_SETTING_BUILD_TYPE_UPPER STREQUAL "MINSIZEREL") + set(_CONAN_SETTING_BUILD_TYPE "MinSizeRel") + endif() +endmacro() + +macro(_conan_check_system_name) + #handle -s os setting + if(CMAKE_SYSTEM_NAME AND NOT CMAKE_SYSTEM_NAME STREQUAL "Generic") + #use default conan os setting if CMAKE_SYSTEM_NAME is not defined + set(CONAN_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}) + if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + set(CONAN_SYSTEM_NAME Macos) + endif() + if(${CMAKE_SYSTEM_NAME} STREQUAL "QNX") + set(CONAN_SYSTEM_NAME Neutrino) + endif() + set(CONAN_SUPPORTED_PLATFORMS Windows Linux Macos Android iOS FreeBSD WindowsStore WindowsCE watchOS tvOS FreeBSD SunOS AIX Arduino Emscripten Neutrino) + list (FIND CONAN_SUPPORTED_PLATFORMS "${CONAN_SYSTEM_NAME}" _index) + if (${_index} GREATER -1) + #check if the cmake system is a conan supported one + set(_CONAN_SETTING_OS ${CONAN_SYSTEM_NAME}) + else() + message(FATAL_ERROR "cmake system ${CONAN_SYSTEM_NAME} is not supported by conan. Use one of ${CONAN_SUPPORTED_PLATFORMS}") + endif() + endif() +endmacro() + +macro(_conan_check_language) + get_property(_languages GLOBAL PROPERTY ENABLED_LANGUAGES) + if (";${_languages};" MATCHES ";CXX;") + set(LANGUAGE CXX) + set(USING_CXX 1) + elseif (";${_languages};" MATCHES ";C;") + set(LANGUAGE C) + set(USING_CXX 0) + else () + message(FATAL_ERROR "Conan: Neither C or C++ was detected as a language for the project. Unabled to detect compiler version.") + endif() +endmacro() + +macro(_conan_detect_compiler) + + conan_parse_arguments(${ARGV}) + + if(ARGUMENTS_ARCH) + set(_CONAN_SETTING_ARCH ${ARGUMENTS_ARCH}) + endif() + + if(USING_CXX) + set(_CONAN_SETTING_COMPILER_CPPSTD ${CMAKE_CXX_STANDARD}) + endif() + + if (${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL GNU) + # using GCC + # TODO: Handle other params + string(REPLACE "." ";" VERSION_LIST ${CMAKE_${LANGUAGE}_COMPILER_VERSION}) + list(GET VERSION_LIST 0 MAJOR) + list(GET VERSION_LIST 1 MINOR) + set(COMPILER_VERSION ${MAJOR}.${MINOR}) + if(${MAJOR} GREATER 4) + set(COMPILER_VERSION ${MAJOR}) + endif() + set(_CONAN_SETTING_COMPILER gcc) + set(_CONAN_SETTING_COMPILER_VERSION ${COMPILER_VERSION}) + if (USING_CXX) + conan_cmake_detect_unix_libcxx(_LIBCXX) + set(_CONAN_SETTING_COMPILER_LIBCXX ${_LIBCXX}) + endif () + elseif (${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL Intel) + string(REPLACE "." ";" VERSION_LIST ${CMAKE_${LANGUAGE}_COMPILER_VERSION}) + list(GET VERSION_LIST 0 MAJOR) + list(GET VERSION_LIST 1 MINOR) + set(COMPILER_VERSION ${MAJOR}.${MINOR}) + set(_CONAN_SETTING_COMPILER intel) + set(_CONAN_SETTING_COMPILER_VERSION ${COMPILER_VERSION}) + if (USING_CXX) + conan_cmake_detect_unix_libcxx(_LIBCXX) + set(_CONAN_SETTING_COMPILER_LIBCXX ${_LIBCXX}) + endif () + elseif (${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL AppleClang) + # using AppleClang + string(REPLACE "." ";" VERSION_LIST ${CMAKE_${LANGUAGE}_COMPILER_VERSION}) + list(GET VERSION_LIST 0 MAJOR) + list(GET VERSION_LIST 1 MINOR) + set(_CONAN_SETTING_COMPILER apple-clang) + set(_CONAN_SETTING_COMPILER_VERSION ${MAJOR}.${MINOR}) + if (USING_CXX) + conan_cmake_detect_unix_libcxx(_LIBCXX) + set(_CONAN_SETTING_COMPILER_LIBCXX ${_LIBCXX}) + endif () + elseif (${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL Clang) + string(REPLACE "." ";" VERSION_LIST ${CMAKE_${LANGUAGE}_COMPILER_VERSION}) + list(GET VERSION_LIST 0 MAJOR) + list(GET VERSION_LIST 1 MINOR) + set(_CONAN_SETTING_COMPILER clang) + set(_CONAN_SETTING_COMPILER_VERSION ${MAJOR}.${MINOR}) + if(APPLE) + cmake_policy(GET CMP0025 APPLE_CLANG_POLICY) + if(NOT APPLE_CLANG_POLICY STREQUAL NEW) + message(STATUS "Conan: APPLE and Clang detected. Assuming apple-clang compiler. Set CMP0025 to avoid it") + set(_CONAN_SETTING_COMPILER apple-clang) + endif() + endif() + if(${_CONAN_SETTING_COMPILER} STREQUAL clang AND ${MAJOR} GREATER 7) + set(_CONAN_SETTING_COMPILER_VERSION ${MAJOR}) + endif() + if (USING_CXX) + conan_cmake_detect_unix_libcxx(_LIBCXX) + set(_CONAN_SETTING_COMPILER_LIBCXX ${_LIBCXX}) + endif () + elseif(${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL MSVC) + set(_VISUAL "Visual Studio") + _get_msvc_ide_version(_VISUAL_VERSION) + if("${_VISUAL_VERSION}" STREQUAL "") + message(FATAL_ERROR "Conan: Visual Studio not recognized") + else() + set(_CONAN_SETTING_COMPILER ${_VISUAL}) + set(_CONAN_SETTING_COMPILER_VERSION ${_VISUAL_VERSION}) + endif() + + if(NOT _CONAN_SETTING_ARCH) + if (MSVC_${LANGUAGE}_ARCHITECTURE_ID MATCHES "64") + set(_CONAN_SETTING_ARCH x86_64) + elseif (MSVC_${LANGUAGE}_ARCHITECTURE_ID MATCHES "^ARM") + message(STATUS "Conan: Using default ARM architecture from MSVC") + set(_CONAN_SETTING_ARCH armv6) + elseif (MSVC_${LANGUAGE}_ARCHITECTURE_ID MATCHES "86") + set(_CONAN_SETTING_ARCH x86) + else () + message(FATAL_ERROR "Conan: Unknown MSVC architecture [${MSVC_${LANGUAGE}_ARCHITECTURE_ID}]") + endif() + endif() + + conan_cmake_detect_vs_runtime(_vs_runtime ${ARGV}) + message(STATUS "Conan: Detected VS runtime: ${_vs_runtime}") + set(_CONAN_SETTING_COMPILER_RUNTIME ${_vs_runtime}) + + if (CMAKE_GENERATOR_TOOLSET) + set(_CONAN_SETTING_COMPILER_TOOLSET ${CMAKE_VS_PLATFORM_TOOLSET}) + elseif(CMAKE_VS_PLATFORM_TOOLSET AND (CMAKE_GENERATOR STREQUAL "Ninja")) + set(_CONAN_SETTING_COMPILER_TOOLSET ${CMAKE_VS_PLATFORM_TOOLSET}) + endif() + else() + message(FATAL_ERROR "Conan: compiler setup not recognized") + endif() + +endmacro() + +function(conan_cmake_settings result) + #message(STATUS "COMPILER " ${CMAKE_CXX_COMPILER}) + #message(STATUS "COMPILER " ${CMAKE_CXX_COMPILER_ID}) + #message(STATUS "VERSION " ${CMAKE_CXX_COMPILER_VERSION}) + #message(STATUS "FLAGS " ${CMAKE_LANG_FLAGS}) + #message(STATUS "LIB ARCH " ${CMAKE_CXX_LIBRARY_ARCHITECTURE}) + #message(STATUS "BUILD TYPE " ${CMAKE_BUILD_TYPE}) + #message(STATUS "GENERATOR " ${CMAKE_GENERATOR}) + #message(STATUS "GENERATOR WIN64 " ${CMAKE_CL_64}) + + message(STATUS "Conan: Automatic detection of conan settings from cmake") + + conan_parse_arguments(${ARGV}) + + _conan_detect_build_type(${ARGV}) + + _conan_check_system_name() + + _conan_check_language() + + _conan_detect_compiler(${ARGV}) + + # If profile is defined it is used + if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND ARGUMENTS_DEBUG_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_DEBUG_PROFILE}) + elseif(CMAKE_BUILD_TYPE STREQUAL "Release" AND ARGUMENTS_RELEASE_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_RELEASE_PROFILE}) + elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" AND ARGUMENTS_RELWITHDEBINFO_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_RELWITHDEBINFO_PROFILE}) + elseif(CMAKE_BUILD_TYPE STREQUAL "MinSizeRel" AND ARGUMENTS_MINSIZEREL_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_MINSIZEREL_PROFILE}) + elseif(ARGUMENTS_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_PROFILE}) + endif() + + foreach(ARG ${_APPLIED_PROFILES}) + set(_SETTINGS ${_SETTINGS} -pr=${ARG}) + endforeach() + foreach(ARG ${ARGUMENTS_PROFILE_BUILD}) + conan_check(VERSION 1.24.0 REQUIRED DETECT_QUIET) + set(_SETTINGS ${_SETTINGS} -pr:b=${ARG}) + endforeach() + + if(NOT _SETTINGS OR ARGUMENTS_PROFILE_AUTO STREQUAL "ALL") + set(ARGUMENTS_PROFILE_AUTO arch build_type compiler compiler.version + compiler.runtime compiler.libcxx compiler.toolset) + endif() + + # remove any manually specified settings from the autodetected settings + foreach(ARG ${ARGUMENTS_SETTINGS}) + string(REGEX MATCH "[^=]*" MANUAL_SETTING "${ARG}") + message(STATUS "Conan: ${MANUAL_SETTING} was added as an argument. Not using the autodetected one.") + list(REMOVE_ITEM ARGUMENTS_PROFILE_AUTO "${MANUAL_SETTING}") + endforeach() + + # Automatic from CMake + foreach(ARG ${ARGUMENTS_PROFILE_AUTO}) + string(TOUPPER ${ARG} _arg_name) + string(REPLACE "." "_" _arg_name ${_arg_name}) + if(_CONAN_SETTING_${_arg_name}) + set(_SETTINGS ${_SETTINGS} -s ${ARG}=${_CONAN_SETTING_${_arg_name}}) + endif() + endforeach() + + foreach(ARG ${ARGUMENTS_SETTINGS}) + set(_SETTINGS ${_SETTINGS} -s ${ARG}) + endforeach() + + message(STATUS "Conan: Settings= ${_SETTINGS}") + + set(${result} ${_SETTINGS} PARENT_SCOPE) +endfunction() + + +function(conan_cmake_detect_unix_libcxx result) + # Take into account any -stdlib in compile options + get_directory_property(compile_options DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMPILE_OPTIONS) + string(GENEX_STRIP "${compile_options}" compile_options) + + # Take into account any _GLIBCXX_USE_CXX11_ABI in compile definitions + get_directory_property(defines DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMPILE_DEFINITIONS) + string(GENEX_STRIP "${defines}" defines) + + foreach(define ${defines}) + if(define MATCHES "_GLIBCXX_USE_CXX11_ABI") + if(define MATCHES "^-D") + set(compile_options ${compile_options} "${define}") + else() + set(compile_options ${compile_options} "-D${define}") + endif() + endif() + endforeach() + + # add additional compiler options ala cmRulePlaceholderExpander::ExpandRuleVariable + set(EXPAND_CXX_COMPILER ${CMAKE_CXX_COMPILER}) + if(CMAKE_CXX_COMPILER_ARG1) + # CMake splits CXX="foo bar baz" into CMAKE_CXX_COMPILER="foo", CMAKE_CXX_COMPILER_ARG1="bar baz" + # without this, ccache, winegcc, or other wrappers might lose all their arguments + separate_arguments(SPLIT_CXX_COMPILER_ARG1 NATIVE_COMMAND ${CMAKE_CXX_COMPILER_ARG1}) + list(APPEND EXPAND_CXX_COMPILER ${SPLIT_CXX_COMPILER_ARG1}) + endif() + + if(CMAKE_CXX_COMPILE_OPTIONS_TARGET AND CMAKE_CXX_COMPILER_TARGET) + # without --target= we may be calling the wrong underlying GCC + list(APPEND EXPAND_CXX_COMPILER "${CMAKE_CXX_COMPILE_OPTIONS_TARGET}${CMAKE_CXX_COMPILER_TARGET}") + endif() + + if(CMAKE_CXX_COMPILE_OPTIONS_EXTERNAL_TOOLCHAIN AND CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN) + list(APPEND EXPAND_CXX_COMPILER "${CMAKE_CXX_COMPILE_OPTIONS_EXTERNAL_TOOLCHAIN}${CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN}") + endif() + + if(CMAKE_CXX_COMPILE_OPTIONS_SYSROOT) + # without --sysroot= we may find the wrong #include + if(CMAKE_SYSROOT_COMPILE) + list(APPEND EXPAND_CXX_COMPILER "${CMAKE_CXX_COMPILE_OPTIONS_SYSROOT}${CMAKE_SYSROOT_COMPILE}") + elseif(CMAKE_SYSROOT) + list(APPEND EXPAND_CXX_COMPILER "${CMAKE_CXX_COMPILE_OPTIONS_SYSROOT}${CMAKE_SYSROOT}") + endif() + endif() + + separate_arguments(SPLIT_CXX_FLAGS NATIVE_COMMAND ${CMAKE_CXX_FLAGS}) + + if(CMAKE_OSX_SYSROOT) + set(xcode_sysroot_option "--sysroot=${CMAKE_OSX_SYSROOT}") + endif() + + execute_process( + COMMAND ${CMAKE_COMMAND} -E echo "#include " + COMMAND ${EXPAND_CXX_COMPILER} ${SPLIT_CXX_FLAGS} -x c++ ${xcode_sysroot_option} ${compile_options} -E -dM - + OUTPUT_VARIABLE string_defines + ) + + if(string_defines MATCHES "#define __GLIBCXX__") + # Allow -D_GLIBCXX_USE_CXX11_ABI=ON/OFF as argument to cmake + if(DEFINED _GLIBCXX_USE_CXX11_ABI) + if(_GLIBCXX_USE_CXX11_ABI) + set(${result} libstdc++11 PARENT_SCOPE) + return() + else() + set(${result} libstdc++ PARENT_SCOPE) + return() + endif() + endif() + + if(string_defines MATCHES "#define _GLIBCXX_USE_CXX11_ABI 1\n") + set(${result} libstdc++11 PARENT_SCOPE) + else() + # Either the compiler is missing the define because it is old, and so + # it can't use the new abi, or the compiler was configured to use the + # old abi by the user or distro (e.g. devtoolset on RHEL/CentOS) + set(${result} libstdc++ PARENT_SCOPE) + endif() + else() + set(${result} libc++ PARENT_SCOPE) + endif() +endfunction() + +function(conan_cmake_detect_vs_runtime result) + + conan_parse_arguments(${ARGV}) + if(ARGUMENTS_BUILD_TYPE) + set(build_type "${ARGUMENTS_BUILD_TYPE}") + elseif(CMAKE_BUILD_TYPE) + set(build_type "${CMAKE_BUILD_TYPE}") + else() + message(FATAL_ERROR "Please specify in command line CMAKE_BUILD_TYPE (-DCMAKE_BUILD_TYPE=Release)") + endif() + + if(build_type) + string(TOUPPER "${build_type}" build_type) + endif() + set(variables CMAKE_CXX_FLAGS_${build_type} CMAKE_C_FLAGS_${build_type} CMAKE_CXX_FLAGS CMAKE_C_FLAGS) + foreach(variable ${variables}) + if(NOT "${${variable}}" STREQUAL "") + string(REPLACE " " ";" flags "${${variable}}") + foreach (flag ${flags}) + if("${flag}" STREQUAL "/MD" OR "${flag}" STREQUAL "/MDd" OR "${flag}" STREQUAL "/MT" OR "${flag}" STREQUAL "/MTd") + string(SUBSTRING "${flag}" 1 -1 runtime) + set(${result} "${runtime}" PARENT_SCOPE) + return() + endif() + endforeach() + endif() + endforeach() + if("${build_type}" STREQUAL "DEBUG") + set(${result} "MDd" PARENT_SCOPE) + else() + set(${result} "MD" PARENT_SCOPE) + endif() +endfunction() + +function(_collect_settings result) + set(ARGUMENTS_PROFILE_AUTO arch build_type compiler compiler.version + compiler.runtime compiler.libcxx compiler.toolset + compiler.cppstd) + foreach(ARG ${ARGUMENTS_PROFILE_AUTO}) + string(TOUPPER ${ARG} _arg_name) + string(REPLACE "." "_" _arg_name ${_arg_name}) + if(_CONAN_SETTING_${_arg_name}) + set(detected_setings ${detected_setings} ${ARG}=${_CONAN_SETTING_${_arg_name}}) + endif() + endforeach() + set(${result} ${detected_setings} PARENT_SCOPE) +endfunction() + +function(conan_cmake_autodetect detected_settings) + _conan_detect_build_type(${ARGV}) + _conan_check_system_name() + _conan_check_language() + _conan_detect_compiler(${ARGV}) + _collect_settings(collected_settings) + set(${detected_settings} ${collected_settings} PARENT_SCOPE) +endfunction() + +macro(conan_parse_arguments) + set(options BASIC_SETUP CMAKE_TARGETS UPDATE KEEP_RPATHS NO_LOAD NO_OUTPUT_DIRS OUTPUT_QUIET NO_IMPORTS SKIP_STD) + set(oneValueArgs CONANFILE ARCH BUILD_TYPE INSTALL_FOLDER CONAN_COMMAND) + set(multiValueArgs DEBUG_PROFILE RELEASE_PROFILE RELWITHDEBINFO_PROFILE MINSIZEREL_PROFILE + PROFILE REQUIRES OPTIONS IMPORTS SETTINGS BUILD ENV GENERATORS PROFILE_AUTO + INSTALL_ARGS CONFIGURATION_TYPES PROFILE_BUILD BUILD_REQUIRES) + cmake_parse_arguments(ARGUMENTS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) +endmacro() + +function(old_conan_cmake_install) + # Calls "conan install" + # Argument BUILD is equivalant to --build={missing, PkgName,...} or + # --build when argument is 'BUILD all' (which builds all packages from source) + # Argument CONAN_COMMAND, to specify the conan path, e.g. in case of running from source + # cmake does not identify conan as command, even if it is +x and it is in the path + conan_parse_arguments(${ARGV}) + + if(CONAN_CMAKE_MULTI) + set(ARGUMENTS_GENERATORS ${ARGUMENTS_GENERATORS} cmake_multi) + else() + set(ARGUMENTS_GENERATORS ${ARGUMENTS_GENERATORS} cmake) + endif() + + set(CONAN_BUILD_POLICY "") + foreach(ARG ${ARGUMENTS_BUILD}) + if(${ARG} STREQUAL "all") + set(CONAN_BUILD_POLICY ${CONAN_BUILD_POLICY} --build) + break() + else() + set(CONAN_BUILD_POLICY ${CONAN_BUILD_POLICY} --build=${ARG}) + endif() + endforeach() + if(ARGUMENTS_CONAN_COMMAND) + set(CONAN_CMD ${ARGUMENTS_CONAN_COMMAND}) + else() + conan_check(REQUIRED) + endif() + set(CONAN_OPTIONS "") + if(ARGUMENTS_CONANFILE) + if(IS_ABSOLUTE ${ARGUMENTS_CONANFILE}) + set(CONANFILE ${ARGUMENTS_CONANFILE}) + else() + set(CONANFILE ${CMAKE_CURRENT_SOURCE_DIR}/${ARGUMENTS_CONANFILE}) + endif() + else() + set(CONANFILE ".") + endif() + foreach(ARG ${ARGUMENTS_OPTIONS}) + set(CONAN_OPTIONS ${CONAN_OPTIONS} -o=${ARG}) + endforeach() + if(ARGUMENTS_UPDATE) + set(CONAN_INSTALL_UPDATE --update) + endif() + if(ARGUMENTS_NO_IMPORTS) + set(CONAN_INSTALL_NO_IMPORTS --no-imports) + endif() + set(CONAN_INSTALL_FOLDER "") + if(ARGUMENTS_INSTALL_FOLDER) + set(CONAN_INSTALL_FOLDER -if=${ARGUMENTS_INSTALL_FOLDER}) + endif() + foreach(ARG ${ARGUMENTS_GENERATORS}) + set(CONAN_GENERATORS ${CONAN_GENERATORS} -g=${ARG}) + endforeach() + foreach(ARG ${ARGUMENTS_ENV}) + set(CONAN_ENV_VARS ${CONAN_ENV_VARS} -e=${ARG}) + endforeach() + set(conan_args install ${CONANFILE} ${settings} ${CONAN_ENV_VARS} ${CONAN_GENERATORS} ${CONAN_BUILD_POLICY} ${CONAN_INSTALL_UPDATE} ${CONAN_INSTALL_NO_IMPORTS} ${CONAN_OPTIONS} ${CONAN_INSTALL_FOLDER} ${ARGUMENTS_INSTALL_ARGS}) + + string (REPLACE ";" " " _conan_args "${conan_args}") + message(STATUS "Conan executing: ${CONAN_CMD} ${_conan_args}") + + if(ARGUMENTS_OUTPUT_QUIET) + execute_process(COMMAND ${CONAN_CMD} ${conan_args} + RESULT_VARIABLE return_code + OUTPUT_VARIABLE conan_output + ERROR_VARIABLE conan_output + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + else() + execute_process(COMMAND ${CONAN_CMD} ${conan_args} + RESULT_VARIABLE return_code + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + endif() + + if(NOT "${return_code}" STREQUAL "0") + message(FATAL_ERROR "Conan install failed='${return_code}'") + endif() + +endfunction() + +function(conan_cmake_install) + if(DEFINED CONAN_COMMAND) + set(CONAN_CMD ${CONAN_COMMAND}) + else() + conan_check(REQUIRED) + endif() + + set(installOptions UPDATE NO_IMPORTS OUTPUT_QUIET ERROR_QUIET) + set(installOneValueArgs PATH_OR_REFERENCE REFERENCE REMOTE LOCKFILE LOCKFILE_OUT LOCKFILE_NODE_ID INSTALL_FOLDER) + set(installMultiValueArgs GENERATOR BUILD ENV ENV_HOST ENV_BUILD OPTIONS_HOST OPTIONS OPTIONS_BUILD PROFILE + PROFILE_HOST PROFILE_BUILD SETTINGS SETTINGS_HOST SETTINGS_BUILD) + cmake_parse_arguments(ARGS "${installOptions}" "${installOneValueArgs}" "${installMultiValueArgs}" ${ARGN}) + foreach(arg ${installOptions}) + if(ARGS_${arg}) + set(${arg} ${${arg}} ${ARGS_${arg}}) + endif() + endforeach() + foreach(arg ${installOneValueArgs}) + if(DEFINED ARGS_${arg}) + if("${arg}" STREQUAL "REMOTE") + set(flag "--remote") + elseif("${arg}" STREQUAL "LOCKFILE") + set(flag "--lockfile") + elseif("${arg}" STREQUAL "LOCKFILE_OUT") + set(flag "--lockfile-out") + elseif("${arg}" STREQUAL "LOCKFILE_NODE_ID") + set(flag "--lockfile-node-id") + elseif("${arg}" STREQUAL "INSTALL_FOLDER") + set(flag "--install-folder") + endif() + set(${arg} ${${arg}} ${flag} ${ARGS_${arg}}) + endif() + endforeach() + foreach(arg ${installMultiValueArgs}) + if(DEFINED ARGS_${arg}) + if("${arg}" STREQUAL "GENERATOR") + set(flag "--generator") + elseif("${arg}" STREQUAL "BUILD") + set(flag "--build") + elseif("${arg}" STREQUAL "ENV") + set(flag "--env") + elseif("${arg}" STREQUAL "ENV_HOST") + set(flag "--env:host") + elseif("${arg}" STREQUAL "ENV_BUILD") + set(flag "--env:build") + elseif("${arg}" STREQUAL "OPTIONS") + set(flag "--options") + elseif("${arg}" STREQUAL "OPTIONS_HOST") + set(flag "--options:host") + elseif("${arg}" STREQUAL "OPTIONS_BUILD") + set(flag "--options:build") + elseif("${arg}" STREQUAL "PROFILE") + set(flag "--profile") + elseif("${arg}" STREQUAL "PROFILE_HOST") + set(flag "--profile:host") + elseif("${arg}" STREQUAL "PROFILE_BUILD") + set(flag "--profile:build") + elseif("${arg}" STREQUAL "SETTINGS") + set(flag "--settings") + elseif("${arg}" STREQUAL "SETTINGS_HOST") + set(flag "--settings:host") + elseif("${arg}" STREQUAL "SETTINGS_BUILD") + set(flag "--settings:build") + endif() + list(LENGTH ARGS_${arg} numargs) + foreach(item ${ARGS_${arg}}) + if(${item} STREQUAL "all" AND ${arg} STREQUAL "BUILD") + set(${arg} "--build") + break() + endif() + set(${arg} ${${arg}} ${flag} ${item}) + endforeach() + endif() + endforeach() + if(DEFINED UPDATE) + set(UPDATE --update) + endif() + if(DEFINED NO_IMPORTS) + set(NO_IMPORTS --no-imports) + endif() + set(install_args install ${PATH_OR_REFERENCE} ${REFERENCE} ${UPDATE} ${NO_IMPORTS} ${REMOTE} ${LOCKFILE} ${LOCKFILE_OUT} ${LOCKFILE_NODE_ID} ${INSTALL_FOLDER} + ${GENERATOR} ${BUILD} ${ENV} ${ENV_HOST} ${ENV_BUILD} ${OPTIONS} ${OPTIONS_HOST} ${OPTIONS_BUILD} + ${PROFILE} ${PROFILE_HOST} ${PROFILE_BUILD} ${SETTINGS} ${SETTINGS_HOST} ${SETTINGS_BUILD}) + + string(REPLACE ";" " " _install_args "${install_args}") + message(STATUS "Conan executing: ${CONAN_CMD} ${_install_args}") + + if(ARGS_OUTPUT_QUIET) + set(OUTPUT_OPT OUTPUT_QUIET) + endif() + if(ARGS_ERROR_QUIET) + set(ERROR_OPT ERROR_QUIET) + endif() + + execute_process(COMMAND ${CONAN_CMD} ${install_args} + RESULT_VARIABLE return_code + ${OUTPUT_OPT} + ${ERROR_OPT} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + + if(NOT "${return_code}" STREQUAL "0") + if (ARGS_ERROR_QUIET) + message(WARNING "Conan install failed='${return_code}'") + else() + message(FATAL_ERROR "Conan install failed='${return_code}'") + endif() + endif() + +endfunction() + +function(conan_cmake_setup_conanfile) + conan_parse_arguments(${ARGV}) + if(ARGUMENTS_CONANFILE) + get_filename_component(_CONANFILE_NAME ${ARGUMENTS_CONANFILE} NAME) + # configure_file will make sure cmake re-runs when conanfile is updated + configure_file(${ARGUMENTS_CONANFILE} ${CMAKE_CURRENT_BINARY_DIR}/${_CONANFILE_NAME}.junk COPYONLY) + file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${_CONANFILE_NAME}.junk) + else() + conan_cmake_generate_conanfile(ON ${ARGV}) + endif() +endfunction() + +function(conan_cmake_configure) + conan_cmake_generate_conanfile(OFF ${ARGV}) +endfunction() + +# Generate, writing in disk a conanfile.txt with the requires, options, and imports +# specified as arguments +# This will be considered as temporary file, generated in CMAKE_CURRENT_BINARY_DIR) +function(conan_cmake_generate_conanfile DEFAULT_GENERATOR) + + conan_parse_arguments(${ARGV}) + + set(_FN "${CMAKE_CURRENT_BINARY_DIR}/conanfile.txt") + file(WRITE ${_FN} "") + + if(DEFINED ARGUMENTS_REQUIRES) + file(APPEND ${_FN} "[requires]\n") + foreach(REQUIRE ${ARGUMENTS_REQUIRES}) + file(APPEND ${_FN} ${REQUIRE} "\n") + endforeach() + endif() + + if (DEFAULT_GENERATOR OR DEFINED ARGUMENTS_GENERATORS) + file(APPEND ${_FN} "[generators]\n") + if (DEFAULT_GENERATOR) + file(APPEND ${_FN} "cmake\n") + endif() + if (DEFINED ARGUMENTS_GENERATORS) + foreach(GENERATOR ${ARGUMENTS_GENERATORS}) + file(APPEND ${_FN} ${GENERATOR} "\n") + endforeach() + endif() + endif() + + if(DEFINED ARGUMENTS_BUILD_REQUIRES) + file(APPEND ${_FN} "[build_requires]\n") + foreach(BUILD_REQUIRE ${ARGUMENTS_BUILD_REQUIRES}) + file(APPEND ${_FN} ${BUILD_REQUIRE} "\n") + endforeach() + endif() + + if(DEFINED ARGUMENTS_IMPORTS) + file(APPEND ${_FN} "[imports]\n") + foreach(IMPORTS ${ARGUMENTS_IMPORTS}) + file(APPEND ${_FN} ${IMPORTS} "\n") + endforeach() + endif() + + if(DEFINED ARGUMENTS_OPTIONS) + file(APPEND ${_FN} "[options]\n") + foreach(OPTION ${ARGUMENTS_OPTIONS}) + file(APPEND ${_FN} ${OPTION} "\n") + endforeach() + endif() + +endfunction() + + +macro(conan_load_buildinfo) + if(CONAN_CMAKE_MULTI) + set(_CONANBUILDINFO conanbuildinfo_multi.cmake) + else() + set(_CONANBUILDINFO conanbuildinfo.cmake) + endif() + if(ARGUMENTS_INSTALL_FOLDER) + set(_CONANBUILDINFOFOLDER ${ARGUMENTS_INSTALL_FOLDER}) + else() + set(_CONANBUILDINFOFOLDER ${CMAKE_CURRENT_BINARY_DIR}) + endif() + # Checks for the existence of conanbuildinfo.cmake, and loads it + # important that it is macro, so variables defined at parent scope + if(EXISTS "${_CONANBUILDINFOFOLDER}/${_CONANBUILDINFO}") + message(STATUS "Conan: Loading ${_CONANBUILDINFO}") + include(${_CONANBUILDINFOFOLDER}/${_CONANBUILDINFO}) + else() + message(FATAL_ERROR "${_CONANBUILDINFO} doesn't exist in ${CMAKE_CURRENT_BINARY_DIR}") + endif() +endmacro() + + +macro(conan_cmake_run) + conan_parse_arguments(${ARGV}) + + if(ARGUMENTS_CONFIGURATION_TYPES AND NOT CMAKE_CONFIGURATION_TYPES) + message(WARNING "CONFIGURATION_TYPES should only be specified for multi-configuration generators") + elseif(ARGUMENTS_CONFIGURATION_TYPES AND ARGUMENTS_BUILD_TYPE) + message(WARNING "CONFIGURATION_TYPES and BUILD_TYPE arguments should not be defined at the same time.") + endif() + + if(CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE AND NOT CONAN_EXPORTED + AND NOT ARGUMENTS_BUILD_TYPE) + set(CONAN_CMAKE_MULTI ON) + if (NOT ARGUMENTS_CONFIGURATION_TYPES) + set(ARGUMENTS_CONFIGURATION_TYPES "Release;Debug") + endif() + message(STATUS "Conan: Using cmake-multi generator") + else() + set(CONAN_CMAKE_MULTI OFF) + endif() + + if(NOT CONAN_EXPORTED) + conan_cmake_setup_conanfile(${ARGV}) + if(CONAN_CMAKE_MULTI) + foreach(CMAKE_BUILD_TYPE ${ARGUMENTS_CONFIGURATION_TYPES}) + set(ENV{CONAN_IMPORT_PATH} ${CMAKE_BUILD_TYPE}) + conan_cmake_settings(settings ${ARGV}) + old_conan_cmake_install(SETTINGS ${settings} ${ARGV}) + endforeach() + set(CMAKE_BUILD_TYPE) + else() + conan_cmake_settings(settings ${ARGV}) + old_conan_cmake_install(SETTINGS ${settings} ${ARGV}) + endif() + endif() + + if (NOT ARGUMENTS_NO_LOAD) + conan_load_buildinfo() + endif() + + if(ARGUMENTS_BASIC_SETUP) + foreach(_option CMAKE_TARGETS KEEP_RPATHS NO_OUTPUT_DIRS SKIP_STD) + if(ARGUMENTS_${_option}) + if(${_option} STREQUAL "CMAKE_TARGETS") + list(APPEND _setup_options "TARGETS") + else() + list(APPEND _setup_options ${_option}) + endif() + endif() + endforeach() + conan_basic_setup(${_setup_options}) + endif() +endmacro() + +macro(conan_check) + # Checks conan availability in PATH + # Arguments REQUIRED, DETECT_QUIET and VERSION are optional + # Example usage: + # conan_check(VERSION 1.0.0 REQUIRED) + set(options REQUIRED DETECT_QUIET) + set(oneValueArgs VERSION) + cmake_parse_arguments(CONAN "${options}" "${oneValueArgs}" "" ${ARGN}) + if(NOT CONAN_DETECT_QUIET) + message(STATUS "Conan: checking conan executable") + endif() + + find_program(CONAN_CMD conan) + if(NOT CONAN_CMD AND CONAN_REQUIRED) + message(FATAL_ERROR "Conan executable not found! Please install conan.") + endif() + if(NOT CONAN_DETECT_QUIET) + message(STATUS "Conan: Found program ${CONAN_CMD}") + endif() + execute_process(COMMAND ${CONAN_CMD} --version + RESULT_VARIABLE return_code + OUTPUT_VARIABLE CONAN_VERSION_OUTPUT + ERROR_VARIABLE CONAN_VERSION_OUTPUT) + + if(NOT "${return_code}" STREQUAL "0") + message(FATAL_ERROR "Conan --version failed='${return_code}'") + endif() + + if(NOT CONAN_DETECT_QUIET) + string(STRIP "${CONAN_VERSION_OUTPUT}" _CONAN_VERSION_OUTPUT) + message(STATUS "Conan: Version found ${_CONAN_VERSION_OUTPUT}") + endif() + + if(DEFINED CONAN_VERSION) + string(REGEX MATCH ".*Conan version ([0-9]+\\.[0-9]+\\.[0-9]+)" FOO + "${CONAN_VERSION_OUTPUT}") + if(${CMAKE_MATCH_1} VERSION_LESS ${CONAN_VERSION}) + message(FATAL_ERROR "Conan outdated. Installed: ${CMAKE_MATCH_1}, \ + required: ${CONAN_VERSION}. Consider updating via 'pip \ + install conan==${CONAN_VERSION}'.") + endif() + endif() +endmacro() + +function(conan_add_remote) + # Adds a remote + # Arguments URL and NAME are required, INDEX, COMMAND and VERIFY_SSL are optional + # Example usage: + # conan_add_remote(NAME bincrafters INDEX 1 + # URL https://api.bintray.com/conan/bincrafters/public-conan + # VERIFY_SSL True) + set(oneValueArgs URL NAME INDEX COMMAND VERIFY_SSL) + cmake_parse_arguments(CONAN "" "${oneValueArgs}" "" ${ARGN}) + + if(DEFINED CONAN_INDEX) + set(CONAN_INDEX_ARG "-i ${CONAN_INDEX}") + endif() + if(DEFINED CONAN_COMMAND) + set(CONAN_CMD ${CONAN_COMMAND}) + else() + conan_check(REQUIRED DETECT_QUIET) + endif() + set(CONAN_VERIFY_SSL_ARG "True") + if(DEFINED CONAN_VERIFY_SSL) + set(CONAN_VERIFY_SSL_ARG ${CONAN_VERIFY_SSL}) + endif() + message(STATUS "Conan: Adding ${CONAN_NAME} remote repository (${CONAN_URL}) verify ssl (${CONAN_VERIFY_SSL_ARG})") + execute_process(COMMAND ${CONAN_CMD} remote add ${CONAN_NAME} ${CONAN_INDEX_ARG} -f ${CONAN_URL} ${CONAN_VERIFY_SSL_ARG} + RESULT_VARIABLE return_code) + if(NOT "${return_code}" STREQUAL "0") + message(FATAL_ERROR "Conan remote failed='${return_code}'") + endif() +endfunction() + +macro(conan_config_install) + # install a full configuration from a local or remote zip file + # Argument ITEM is required, arguments TYPE, SOURCE, TARGET and VERIFY_SSL are optional + # Example usage: + # conan_config_install(ITEM https://github.com/conan-io/cmake-conan.git + # TYPE git SOURCE source-folder TARGET target-folder VERIFY_SSL false) + set(oneValueArgs ITEM TYPE SOURCE TARGET VERIFY_SSL) + set(multiValueArgs ARGS) + cmake_parse_arguments(CONAN "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + find_program(CONAN_CMD conan) + if(NOT CONAN_CMD AND CONAN_REQUIRED) + message(FATAL_ERROR "Conan executable not found!") + endif() + + if(DEFINED CONAN_VERIFY_SSL) + set(CONAN_VERIFY_SSL_ARG "--verify-ssl=${CONAN_VERIFY_SSL}") + endif() + + if(DEFINED CONAN_TYPE) + set(CONAN_TYPE_ARG "--type=${CONAN_TYPE}") + endif() + + if(DEFINED CONAN_ARGS) + set(CONAN_ARGS_ARGS "--args=\"${CONAN_ARGS}\"") + endif() + + if(DEFINED CONAN_SOURCE) + set(CONAN_SOURCE_ARGS "--source-folder=${CONAN_SOURCE}") + endif() + + if(DEFINED CONAN_TARGET) + set(CONAN_TARGET_ARGS "--target-folder=${CONAN_TARGET}") + endif() + + set (CONAN_CONFIG_INSTALL_ARGS ${CONAN_VERIFY_SSL_ARG} + ${CONAN_TYPE_ARG} + ${CONAN_ARGS_ARGS} + ${CONAN_SOURCE_ARGS} + ${CONAN_TARGET_ARGS}) + + message(STATUS "Conan: Installing config from ${CONAN_ITEM}") + execute_process(COMMAND ${CONAN_CMD} config install ${CONAN_ITEM} ${CONAN_CONFIG_INSTALL_ARGS} + RESULT_VARIABLE return_code) + if(NOT "${return_code}" STREQUAL "0") + message(FATAL_ERROR "Conan config failed='${return_code}'") + endif() +endmacro() diff --git a/cmake/kagomeConfig.cmake.in b/cmake/kagomeConfig.cmake.in index 7da66e5ac8..a3562405c8 100644 --- a/cmake/kagomeConfig.cmake.in +++ b/cmake/kagomeConfig.cmake.in @@ -20,6 +20,12 @@ if (KAGOME_ENABLE_WAVM) find_dependency(WAVM REQUIRED) endif() +option(KAGOME_ENABLE_WASMEDGE "Include WasmEdge webassembly executor dependency" OFF) +if (KAGOME_ENABLE_WASMEDGE) + find_dependency(LLVM REQUIRED) + find_dependency(WasmEdge REQUIRED) +endif() + option(KAGOME_ENABLE_BINARYEN "Include Binaryen webassembly executor dependency" ON) if (KAGOME_ENABLE_BINARYEN) find_dependency(binaryen REQUIRED) diff --git a/conanfile.txt b/conanfile.txt deleted file mode 100644 index e26a53ab8a..0000000000 --- a/conanfile.txt +++ /dev/null @@ -1,11 +0,0 @@ -[generators] -cmake -cmake_find_package_multi - -[requires] -WasmEdge/0.8.1@sanblch/stable - -[options] -rocksdb:with_snappy=True - -[imports] diff --git a/core/host_api/CMakeLists.txt b/core/host_api/CMakeLists.txt index 7e17ac3df6..69415972dd 100644 --- a/core/host_api/CMakeLists.txt +++ b/core/host_api/CMakeLists.txt @@ -16,6 +16,6 @@ target_link_libraries(host_api storage_extension child_storage_extension offchain_extension - CONAN_PKG::WasmEdge + WasmEdge::WasmEdge ) kagome_install(host_api) diff --git a/core/host_api/impl/host_api_impl.cpp b/core/host_api/impl/host_api_impl.cpp index a5c51705d0..add377c0cc 100644 --- a/core/host_api/impl/host_api_impl.cpp +++ b/core/host_api/impl/host_api_impl.cpp @@ -14,7 +14,6 @@ #include "crypto/sr25519/sr25519_provider_impl.hpp" #include "host_api/impl/offchain_extension.hpp" #include "runtime/trie_storage_provider.hpp" -#include "runtime/wasmedge/register_host_api.hpp" namespace kagome::host_api { diff --git a/core/log/configurator.cpp b/core/log/configurator.cpp index 39a644f37d..9d59b0376a 100644 --- a/core/log/configurator.cpp +++ b/core/log/configurator.cpp @@ -56,6 +56,7 @@ namespace kagome::log { children: - name: voting_round - name: runtime + level: trace children: - name: runtime_api - name: host_api diff --git a/core/runtime/memory.hpp b/core/runtime/memory.hpp index 4beea1cb8f..364af5f120 100644 --- a/core/runtime/memory.hpp +++ b/core/runtime/memory.hpp @@ -19,7 +19,7 @@ namespace kagome::runtime { constexpr inline size_t kInitialMemorySize = []() { using kagome::common::literals::operator""_MB; - return 5_MB; + return 50_MB; }(); // according to $3.1.2.1 in the Polkadot Host Spec diff --git a/core/runtime/wasmedge/CMakeLists.txt b/core/runtime/wasmedge/CMakeLists.txt index c9cf825a2c..f213053421 100644 --- a/core/runtime/wasmedge/CMakeLists.txt +++ b/core/runtime/wasmedge/CMakeLists.txt @@ -10,7 +10,7 @@ add_library(wasmedge_wasm_memory target_link_libraries(wasmedge_wasm_memory buffer logger - CONAN_PKG::WasmEdge + WasmEdge::WasmEdge memory_allocator ) kagome_install(wasmedge_wasm_memory) diff --git a/core/runtime/wasmedge/instance_environment_factory.cpp b/core/runtime/wasmedge/instance_environment_factory.cpp index 8b25dae1d5..e24363f2e1 100644 --- a/core/runtime/wasmedge/instance_environment_factory.cpp +++ b/core/runtime/wasmedge/instance_environment_factory.cpp @@ -6,7 +6,7 @@ #include "runtime/wasmedge/instance_environment_factory.hpp" #include "host_api/host_api_factory.hpp" -#include "host_api/impl/host_api_impl.hpp" +#include "host_api/host_api.hpp" #include "runtime/common/trie_storage_provider_impl.hpp" #include "runtime/wasmedge/core_api_factory_impl.hpp" #include "runtime/wasmedge/memory_provider.hpp" diff --git a/core/runtime/wasmedge/memory_impl.cpp b/core/runtime/wasmedge/memory_impl.cpp index 980ffb7ae4..9070ee484b 100644 --- a/core/runtime/wasmedge/memory_impl.cpp +++ b/core/runtime/wasmedge/memory_impl.cpp @@ -9,7 +9,7 @@ #include "runtime/common/memory_allocator.hpp" #include "runtime/ptr_size.hpp" -#include +#include namespace { template diff --git a/core/runtime/wasmedge/memory_provider.cpp b/core/runtime/wasmedge/memory_provider.cpp index 90676f3fa9..0ae1648271 100644 --- a/core/runtime/wasmedge/memory_provider.cpp +++ b/core/runtime/wasmedge/memory_provider.cpp @@ -8,7 +8,7 @@ #include "runtime/common/memory_allocator.hpp" #include "runtime/wasmedge/memory_impl.hpp" -#include +#include #include namespace kagome::runtime::wasmedge { @@ -27,8 +27,9 @@ namespace kagome::runtime::wasmedge { void WasmedgeMemoryProvider::setExternalInterface( WasmEdge_ImportObjectContext *imp_obj) { - WasmEdge_Limit MemoryLimit = {.HasMax = false, .Min = 500, .Max = 500}; - mem_ctx_ = WasmEdge_MemoryInstanceCreate(MemoryLimit); + WasmEdge_Limit MemoryLimit = {.HasMax = true, .Min = 400, .Max = 500}; + WasmEdge_MemoryTypeContext *MemTypeCxt = WasmEdge_MemoryTypeCreate(MemoryLimit); + mem_ctx_ = WasmEdge_MemoryInstanceCreate(MemTypeCxt); WasmEdge_String MemoryName = WasmEdge_StringCreateByCString("memory"); WasmEdge_ImportObjectAddMemory(imp_obj, MemoryName, mem_ctx_); WasmEdge_StringDelete(MemoryName); diff --git a/core/runtime/wasmedge/module_impl.cpp b/core/runtime/wasmedge/module_impl.cpp index 9f3c9f05bd..6b7c3e2c41 100644 --- a/core/runtime/wasmedge/module_impl.cpp +++ b/core/runtime/wasmedge/module_impl.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include "common/mp_utils.hpp" #include "crypto/sha/sha256.hpp" @@ -40,7 +40,7 @@ namespace kagome::runtime::wasmedge { auto ConfCtx = WasmEdge_ConfigureCreate(); WasmEdge_ConfigureCompilerSetOptimizationLevel( - ConfCtx, WasmEdge_CompilerOptimizationLevel_O3); + ConfCtx, WasmEdge_CompilerOptimizationLevel_O0); auto CompilerCtx = WasmEdge_CompilerCreate(ConfCtx); auto hash = crypto::sha256(code); diff --git a/core/runtime/wasmedge/module_instance_impl.cpp b/core/runtime/wasmedge/module_instance_impl.cpp index 571dbe6620..bb7f0fbc95 100644 --- a/core/runtime/wasmedge/module_instance_impl.cpp +++ b/core/runtime/wasmedge/module_instance_impl.cpp @@ -5,12 +5,12 @@ #include "runtime/wasmedge/module_instance_impl.hpp" -#include "host_api/impl/host_api_impl.hpp" +#include "host_api/host_api.hpp" #include "runtime/wasmedge/memory_provider.hpp" #include "runtime/wasmedge/module_impl.hpp" #include "runtime/wasmedge/register_host_api.hpp" -#include +#include namespace kagome::runtime::wasmedge { @@ -24,12 +24,12 @@ namespace kagome::runtime::wasmedge { // auto Res = WasmEdge_VMRegisterModuleFromASTModule( // rei_, ModuleName, parent_->ast()); WasmEdge_ImportObjectContext *ImpObj = - WasmEdge_ImportObjectCreate(ModuleName, env_.host_api.get()); + WasmEdge_ImportObjectCreate(ModuleName); // WasmEdge_ImportObjectContext *ImpObj = // WasmEdge_VMGetImportModuleContext(rei_, // WasmEdge_HostRegistration_WasmEdge_Process); WasmEdge_StringDelete(ModuleName); - register_host_api(ImpObj); + register_host_api(ImpObj, env_.host_api.get()); dynamic_cast(env_.memory_provider.get()) ->setExternalInterface(ImpObj); auto Res = WasmEdge_VMRegisterModuleFromImport(rei_, ImpObj); @@ -59,7 +59,8 @@ namespace kagome::runtime::wasmedge { auto GlobalName = WasmEdge_StringCreateByCString(name.data()); auto store = WasmEdge_VMGetStoreContext(rei_); auto res = WasmEdge_StoreFindGlobal(store, GlobalName); - auto type = WasmEdge_GlobalInstanceGetValType(res); + auto type_cxt = WasmEdge_GlobalInstanceGetGlobalType(res); + auto type = WasmEdge_GlobalTypeGetValType(type_cxt); auto val = WasmEdge_GlobalInstanceGetValue(res); WasmEdge_StringDelete(GlobalName); switch (type) { diff --git a/core/runtime/wasmedge/register_host_api.hpp b/core/runtime/wasmedge/register_host_api.hpp index 8dfd96bdc3..fa07060c1c 100644 --- a/core/runtime/wasmedge/register_host_api.hpp +++ b/core/runtime/wasmedge/register_host_api.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -214,116 +214,83 @@ namespace { ParamList, sizeof...(Args), ReturnList, r); } }; + + using kagome::host_api::HostApi; } // namespace template void registerHostApiFunc(const std::string &name, - WasmEdge_ImportObjectContext *ImpObj) { + WasmEdge_ImportObjectContext *ImpObj, + HostApi *host_api) { WasmEdge_FunctionTypeContext *HostFType = HostApiFunc::type(); - WasmEdge_HostFunctionContext *HostFunc = WasmEdge_HostFunctionCreate( - HostFType, HostApiFunc::fun(), 0); + WasmEdge_FunctionInstanceContext *HostFunc = WasmEdge_FunctionInstanceCreate( + HostFType, HostApiFunc::fun(), host_api, 0); WasmEdge_FunctionTypeDelete(HostFType); WasmEdge_String HostFuncName = WasmEdge_StringCreateByCString(name.c_str()); - WasmEdge_ImportObjectAddHostFunction(ImpObj, HostFuncName, HostFunc); + WasmEdge_ImportObjectAddFunction(ImpObj, HostFuncName, HostFunc); WasmEdge_StringDelete(HostFuncName); } -#define REGISTER_HOST_API_FUNC(class, name, obj) \ - registerHostApiFunc<&class ::name>(#name, obj) - -namespace { - using kagome::host_api::HostApi; - using kagome::host_api::HostApiImpl; -} // namespace +#define REGISTER_HOST_API_FUNC(name) \ + registerHostApiFunc<&HostApi ::name>(#name, ImpObj, host_api); -inline void register_host_api(WasmEdge_ImportObjectContext *ImpObj) { - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_trie_blake2_256_ordered_root_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_offchain_index_set_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_logging_log_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_ed25519_generate_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_ed25519_verify_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_finish_batch_verify_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_secp256k1_ecdsa_recover_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_secp256k1_ecdsa_recover_version_2, ImpObj); - REGISTER_HOST_API_FUNC( - HostApi, ext_crypto_secp256k1_ecdsa_recover_compressed_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApi, ext_crypto_secp256k1_ecdsa_recover_compressed_version_2, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_sr25519_generate_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_sr25519_public_keys_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_sr25519_sign_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_sr25519_verify_version_2, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_crypto_start_batch_verify_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_blake2_128_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_blake2_256_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_keccak_256_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_twox_128_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_hashing_twox_64_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_free_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_allocator_malloc_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_hex_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_num_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_misc_print_utf8_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_misc_runtime_version_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_is_validator_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_local_storage_clear_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, - ext_offchain_local_storage_compare_and_set_version_1, - ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_local_storage_get_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_local_storage_set_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_network_state_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_random_seed_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_offchain_submit_transaction_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_offchain_timestamp_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_append_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_changes_root_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_clear_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_clear_prefix_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_clear_prefix_version_2, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_commit_transaction_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_exists_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_get_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_next_key_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_read_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_rollback_transaction_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_root_version_1, ImpObj); - REGISTER_HOST_API_FUNC(HostApiImpl, ext_storage_set_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_storage_start_transaction_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_default_child_storage_clear_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_default_child_storage_get_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_default_child_storage_next_key_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_default_child_storage_set_version_1, ImpObj); - REGISTER_HOST_API_FUNC( - HostApiImpl, ext_default_child_storage_storage_kill_version_1, ImpObj); +inline void register_host_api(WasmEdge_ImportObjectContext *ImpObj, + HostApi *host_api) { + REGISTER_HOST_API_FUNC(ext_allocator_free_version_1); + REGISTER_HOST_API_FUNC(ext_allocator_malloc_version_1); + REGISTER_HOST_API_FUNC(ext_crypto_ed25519_generate_version_1); + REGISTER_HOST_API_FUNC(ext_crypto_ed25519_verify_version_1); + REGISTER_HOST_API_FUNC(ext_crypto_finish_batch_verify_version_1); + REGISTER_HOST_API_FUNC( + ext_crypto_secp256k1_ecdsa_recover_compressed_version_1); + REGISTER_HOST_API_FUNC( + ext_crypto_secp256k1_ecdsa_recover_compressed_version_2); + REGISTER_HOST_API_FUNC(ext_crypto_secp256k1_ecdsa_recover_version_1); + REGISTER_HOST_API_FUNC(ext_crypto_secp256k1_ecdsa_recover_version_2); + REGISTER_HOST_API_FUNC(ext_crypto_sr25519_generate_version_1); + REGISTER_HOST_API_FUNC(ext_crypto_sr25519_public_keys_version_1); + REGISTER_HOST_API_FUNC(ext_crypto_sr25519_sign_version_1); + REGISTER_HOST_API_FUNC(ext_crypto_sr25519_verify_version_2); + REGISTER_HOST_API_FUNC(ext_crypto_start_batch_verify_version_1); + REGISTER_HOST_API_FUNC(ext_default_child_storage_clear_version_1); + REGISTER_HOST_API_FUNC(ext_default_child_storage_get_version_1); + REGISTER_HOST_API_FUNC(ext_default_child_storage_next_key_version_1); + REGISTER_HOST_API_FUNC(ext_default_child_storage_set_version_1); + // REGISTER_HOST_API_FUNC(ext_default_child_storage_storage_kill_version_1); + REGISTER_HOST_API_FUNC(ext_hashing_blake2_128_version_1); + REGISTER_HOST_API_FUNC(ext_hashing_blake2_256_version_1); + REGISTER_HOST_API_FUNC(ext_hashing_keccak_256_version_1); + REGISTER_HOST_API_FUNC(ext_hashing_twox_128_version_1); + REGISTER_HOST_API_FUNC(ext_hashing_twox_64_version_1); + REGISTER_HOST_API_FUNC(ext_logging_log_version_1); + REGISTER_HOST_API_FUNC(ext_misc_print_hex_version_1); + REGISTER_HOST_API_FUNC(ext_misc_print_num_version_1); + REGISTER_HOST_API_FUNC(ext_misc_print_utf8_version_1); + REGISTER_HOST_API_FUNC(ext_misc_runtime_version_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_index_set_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_is_validator_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_local_storage_clear_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_local_storage_compare_and_set_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_local_storage_get_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_local_storage_set_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_network_state_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_random_seed_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_submit_transaction_version_1); + REGISTER_HOST_API_FUNC(ext_offchain_timestamp_version_1); + REGISTER_HOST_API_FUNC(ext_storage_append_version_1); + REGISTER_HOST_API_FUNC(ext_storage_changes_root_version_1); + REGISTER_HOST_API_FUNC(ext_storage_clear_prefix_version_1); + REGISTER_HOST_API_FUNC(ext_storage_clear_prefix_version_2); + REGISTER_HOST_API_FUNC(ext_storage_clear_version_1); + REGISTER_HOST_API_FUNC(ext_storage_commit_transaction_version_1); + REGISTER_HOST_API_FUNC(ext_storage_exists_version_1); + REGISTER_HOST_API_FUNC(ext_storage_get_version_1); + REGISTER_HOST_API_FUNC(ext_storage_next_key_version_1); + REGISTER_HOST_API_FUNC(ext_storage_read_version_1); + REGISTER_HOST_API_FUNC(ext_storage_rollback_transaction_version_1); + REGISTER_HOST_API_FUNC(ext_storage_root_version_1); + REGISTER_HOST_API_FUNC(ext_storage_set_version_1); + REGISTER_HOST_API_FUNC(ext_storage_start_transaction_version_1); + REGISTER_HOST_API_FUNC(ext_trie_blake2_256_ordered_root_version_1); } From 24f482df9dbe810aed4de3b6de345adf7a328513 Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 21 Aug 2023 11:21:22 +0300 Subject: [PATCH 15/43] Remove weird optional locks --- core/storage/trie_pruner/impl/trie_pruner_impl.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp index faf4bb89ab..b1d6335fce 100644 --- a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp +++ b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp @@ -330,7 +330,6 @@ namespace kagome::storage::trie_pruner { outcome::result TriePrunerImpl::addNewState( const storage::trie::RootHash &state_root, trie::StateVersion version) { - std::optional> lock; OUTCOME_TRY(trie, serializer_->retrieveTrie(state_root)); OUTCOME_TRY(addNewStateWith(*trie, version)); return outcome::success(); @@ -338,7 +337,6 @@ namespace kagome::storage::trie_pruner { outcome::result TriePrunerImpl::addNewState( const trie::PolkadotTrie &new_trie, trie::StateVersion version) { - std::optional> lock; OUTCOME_TRY(addNewStateWith(new_trie, version)); return outcome::success(); } From b05e9c76986355df3e9ec52e6459f88d55a69a57 Mon Sep 17 00:00:00 2001 From: Harrm Date: Tue, 22 Aug 2023 15:02:31 +0300 Subject: [PATCH 16/43] Fast pruner startup --- .../trie/impl/persistent_trie_batch_impl.cpp | 2 +- .../trie_pruner/impl/trie_pruner_impl.cpp | 27 ++++++++++++++++++- .../storage/trie_pruner/trie_pruner_test.cpp | 3 +++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/core/storage/trie/impl/persistent_trie_batch_impl.cpp b/core/storage/trie/impl/persistent_trie_batch_impl.cpp index 95ebcd527a..1ad4e76f2e 100644 --- a/core/storage/trie/impl/persistent_trie_batch_impl.cpp +++ b/core/storage/trie/impl/persistent_trie_batch_impl.cpp @@ -42,9 +42,9 @@ namespace kagome::storage::trie { outcome::result PersistentTrieBatchImpl::commit( StateVersion version) { + OUTCOME_TRY(state_pruner_->addNewState(*trie_, version)); OUTCOME_TRY(commitChildren(version)); OUTCOME_TRY(root, serializer_->storeTrie(*trie_, version)); - OUTCOME_TRY(state_pruner_->addNewState(*trie_, version)); SL_TRACE_FUNC_CALL(logger_, root); return std::move(root); } diff --git a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp index b1d6335fce..d46b85d423 100644 --- a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp +++ b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp @@ -337,6 +337,17 @@ namespace kagome::storage::trie_pruner { outcome::result TriePrunerImpl::addNewState( const trie::PolkadotTrie &new_trie, trie::StateVersion version) { + /*EncoderCache encoder{*codec_, logger_}; + + OUTCOME_TRY(root_hash, + encoder.getMerkleValue(*new_trie.getRoot(), version)); + OUTCOME_TRY(root_in_storage, + trie_storage_->contains(root_hash.asHash().value())); + if(root_in_storage) { + SL_WARN(logger_, "Indexing a trie already stored in DB. Root: {}", + root_hash.asHash()); + }*/ + OUTCOME_TRY(addNewStateWith(new_trie, version)); return outcome::success(); } @@ -371,7 +382,19 @@ namespace kagome::storage::trie_pruner { while (!queued_nodes.empty()) { auto [node, hash] = queued_nodes.back(); queued_nodes.pop_back(); - const size_t ref_count = ++ref_count_[hash]; + auto &ref_count = ref_count_[hash]; + if (ref_count == 0) { + OUTCOME_TRY(hash_is_in_storage, trie_storage_->contains(hash)); + if (hash_is_in_storage) { + // the node is present in storage but pruner has not indexed it + // because pruner has been initialized on a newer state + SL_TRACE(logger_, + "Node {} is already in storage, increase ref count", + hash.toHex()); + ref_count++; + } + } + ref_count++; SL_TRACE(logger_, "Add node {}, ref count {}", hash.toHex(), ref_count); referenced_nodes_num++; @@ -394,6 +417,8 @@ namespace kagome::storage::trie_pruner { OUTCOME_TRY(child, serializer_->retrieveNode(opaque_child)); OUTCOME_TRY(child_merkle_val, encoder.getMerkleValue(*child, version)); + // otherwise it is not stored as a separated node, but as a part of + // the branch if (child_merkle_val.isHash()) { SL_TRACE(logger_, "Queue child {}", child_merkle_val.asBuffer()); queued_nodes.push_back({child, *child_merkle_val.asHash()}); diff --git a/test/core/storage/trie_pruner/trie_pruner_test.cpp b/test/core/storage/trie_pruner/trie_pruner_test.cpp index 411a6bbad6..9ff3408ee9 100644 --- a/test/core/storage/trie_pruner/trie_pruner_test.cpp +++ b/test/core/storage/trie_pruner/trie_pruner_test.cpp @@ -625,6 +625,9 @@ TEST_F(TriePrunerTest, RestoreStateFromGenesis) { ON_CALL(*block_tree, getChildren(_)) .WillByDefault(Return(std::vector{})); + ON_CALL(*block_tree, bestLeaf()) + .WillByDefault(Return(BlockInfo{6, hash_from_header(headers.at(6))})); + auto mock_block = [&](unsigned int number) { auto str_number = std::to_string(number); From 006a4ce774b3a2ab9a3867aa2f2895048732119b Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 24 Aug 2023 00:00:07 +0300 Subject: [PATCH 17/43] Add option for thorough pruning --- core/application/app_configuration.hpp | 2 ++ core/application/impl/app_configuration_impl.cpp | 9 ++++++++- core/application/impl/app_configuration_impl.hpp | 4 ++++ .../trie_pruner/impl/trie_pruner_impl.cpp | 16 +++------------- .../trie_pruner/impl/trie_pruner_impl.hpp | 1 + 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/core/application/app_configuration.hpp b/core/application/app_configuration.hpp index 1c81852401..fbab3f4557 100644 --- a/core/application/app_configuration.hpp +++ b/core/application/app_configuration.hpp @@ -256,6 +256,8 @@ namespace kagome::application { virtual bool shouldPruneDiscardedStates() const = 0; + virtual bool enableThoroughPruning() const = 0; + /** * @return database state cache size in MiB */ diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index 269e775c9a..3c6fd7a2a4 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -249,7 +249,7 @@ namespace kagome::application { recovery_state_{def_block_to_recover}, db_cache_size_{def_db_cache_size}, state_pruning_depth_{} { - SL_INFO(logger_, "Soramitsu Kagome started. Version: {} ", buildVersion()); + SL_INFO(logger_, "Kagome started. Version: {} ", buildVersion()); } fs::path AppConfigurationImpl::chainSpecPath() const { @@ -791,6 +791,7 @@ namespace kagome::application { ("enable-offchain-indexing", po::value(), "enable Offchain Indexing API, which allow block import to write to offchain DB)") ("recovery", po::value(), "recovers block storage to state after provided block presented by number or hash, and stop after that") ("state-pruning", po::value()->default_value("archive"), "state pruning policy. 'archive', 'prune-discarded', or the number of finalized blocks to keep.") + ("enable-thorough-pruning", po::bool_switch(), "Makes trie node pruner more efficient, but the node starts slowly") ; po::options_description network_desc("Network options"); @@ -880,6 +881,8 @@ namespace kagome::application { } if (vm.count("help") > 0) { + std::cout + << "Available subcommands: storage-explorer db-editor benchmark\n"; std::cout << desc << std::endl; return false; } @@ -1468,6 +1471,10 @@ namespace kagome::application { return false; } } + + if (find_argument(vm, "enable-thorough-pruning")) { + enable_thorough_pruning_ = true; + } } // if something wrong with config print help message diff --git a/core/application/impl/app_configuration_impl.hpp b/core/application/impl/app_configuration_impl.hpp index a1ff4ef520..22da1fb49f 100644 --- a/core/application/impl/app_configuration_impl.hpp +++ b/core/application/impl/app_configuration_impl.hpp @@ -190,6 +190,9 @@ namespace kagome::application { bool shouldPruneDiscardedStates() const override { return state_pruning_depth_.has_value() || prune_discarded_states_; } + bool enableThoroughPruning() const override { + return enable_thorough_pruning_; + } std::optional devMnemonicPhrase() const override { if (dev_mnemonic_phrase_) { return *dev_mnemonic_phrase_; @@ -345,6 +348,7 @@ namespace kagome::application { uint32_t db_cache_size_; std::optional state_pruning_depth_; bool prune_discarded_states_ = false; + bool enable_thorough_pruning_ = false; std::optional dev_mnemonic_phrase_; std::string node_wss_pem_; std::optional benchmark_config_; diff --git a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp index d46b85d423..cba21c8174 100644 --- a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp +++ b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp @@ -70,7 +70,8 @@ namespace kagome::storage::trie_pruner { codec_{codec}, storage_{storage}, hasher_{hasher}, - pruning_depth_{config->statePruningDepth()} { + pruning_depth_{config->statePruningDepth()}, + thorough_pruning_{config->enableThoroughPruning()} { BOOST_ASSERT(trie_storage_ != nullptr); BOOST_ASSERT(serializer_ != nullptr); BOOST_ASSERT(codec_ != nullptr); @@ -337,17 +338,6 @@ namespace kagome::storage::trie_pruner { outcome::result TriePrunerImpl::addNewState( const trie::PolkadotTrie &new_trie, trie::StateVersion version) { - /*EncoderCache encoder{*codec_, logger_}; - - OUTCOME_TRY(root_hash, - encoder.getMerkleValue(*new_trie.getRoot(), version)); - OUTCOME_TRY(root_in_storage, - trie_storage_->contains(root_hash.asHash().value())); - if(root_in_storage) { - SL_WARN(logger_, "Indexing a trie already stored in DB. Root: {}", - root_hash.asHash()); - }*/ - OUTCOME_TRY(addNewStateWith(new_trie, version)); return outcome::success(); } @@ -383,7 +373,7 @@ namespace kagome::storage::trie_pruner { auto [node, hash] = queued_nodes.back(); queued_nodes.pop_back(); auto &ref_count = ref_count_[hash]; - if (ref_count == 0) { + if (ref_count == 0 && !thorough_pruning_) { OUTCOME_TRY(hash_is_in_storage, trie_storage_->contains(hash)); if (hash_is_in_storage) { // the node is present in storage but pruner has not indexed it diff --git a/core/storage/trie_pruner/impl/trie_pruner_impl.hpp b/core/storage/trie_pruner/impl/trie_pruner_impl.hpp index e11dae80b3..0fabbfec80 100644 --- a/core/storage/trie_pruner/impl/trie_pruner_impl.hpp +++ b/core/storage/trie_pruner/impl/trie_pruner_impl.hpp @@ -143,6 +143,7 @@ namespace kagome::storage::trie_pruner { std::shared_ptr hasher_; const std::optional pruning_depth_{}; + const bool thorough_pruning_{false}; log::Logger logger_ = log::createLogger("TriePruner", "trie_pruner"); }; From 8f33d1f7dbc5f30328f423f6aeca9928f3999324 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 24 Aug 2023 13:22:55 +0300 Subject: [PATCH 18/43] Separate trie nodes and values into separate spaces --- core/injector/application_injector.cpp | 24 ++++-- core/runtime/common/memory_allocator.cpp | 3 +- core/storage/rocksdb/rocksdb_spaces.cpp | 1 + core/storage/spaces.hpp | 1 + .../trie/impl/trie_storage_backend_impl.cpp | 11 ++- .../trie/impl/trie_storage_backend_impl.hpp | 10 ++- .../serialization/trie_serializer_impl.cpp | 15 ++-- .../serialization/trie_serializer_impl.hpp | 9 ++- core/storage/trie/trie_storage_backend.hpp | 15 +++- .../trie_pruner/impl/trie_pruner_impl.cpp | 56 ++++++++----- .../trie_pruner/impl/trie_pruner_impl.hpp | 12 ++- core/utils/kagome_db_editor.cpp | 81 +++++++++++-------- 12 files changed, 155 insertions(+), 83 deletions(-) diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index f31fc979c8..8b2b9bd388 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -189,11 +189,11 @@ namespace { using injector::bind_by_lambda; + template // NodeTag or Value Tag sptr get_trie_storage_backend( - sptr spaced_storage) { - auto storage = spaced_storage->getSpace(storage::Space::kTrieNode); - auto backend = - std::make_shared(storage); + Tag tag, sptr spaced_storage) { + auto backend = std::make_shared( + tag, spaced_storage); return backend; } @@ -719,11 +719,23 @@ namespace { di::bind.template to(), bind_by_lambda( [](const auto &injector) { return get_thread_pool(injector); }), - bind_by_lambda( + bind_by_lambda( [](const auto &injector) { auto storage = injector.template create>(); - return get_trie_storage_backend(storage); + return get_trie_storage_backend( + storage::trie::TrieStorageBackendImpl::NodeTag{}, + storage + ); + }), + bind_by_lambda( + [](const auto &injector) { + auto storage = + injector.template create>(); + return get_trie_storage_backend( + storage::trie::TrieStorageBackendImpl::ValueTag{}, + storage + ); }), bind_by_lambda([](const auto &injector) { diff --git a/core/runtime/common/memory_allocator.cpp b/core/runtime/common/memory_allocator.cpp index c6db30da29..052c9c38d4 100644 --- a/core/runtime/common/memory_allocator.cpp +++ b/core/runtime/common/memory_allocator.cpp @@ -101,8 +101,7 @@ namespace kagome::runtime { } resize(new_size); - auto current_size = memory_.getSize(); - BOOST_ASSERT(current_size >= new_size); + BOOST_ASSERT(memory_.getSize() >= new_size); return allocate(allocation_sz); } diff --git a/core/storage/rocksdb/rocksdb_spaces.cpp b/core/storage/rocksdb/rocksdb_spaces.cpp index 8757ff85e1..b0e3df5dbc 100644 --- a/core/storage/rocksdb/rocksdb_spaces.cpp +++ b/core/storage/rocksdb/rocksdb_spaces.cpp @@ -19,6 +19,7 @@ namespace kagome::storage { "block_body", "justification", "trie_node", + "trie_value", "dispute_data", }; static_assert(kNames.size() == Space::kTotal - 1); diff --git a/core/storage/spaces.hpp b/core/storage/spaces.hpp index 76bc021423..ce97adea03 100644 --- a/core/storage/spaces.hpp +++ b/core/storage/spaces.hpp @@ -20,6 +20,7 @@ namespace kagome::storage { kBlockBody, kJustification, kTrieNode, + kTrieValue, kDisputeData, kTotal diff --git a/core/storage/trie/impl/trie_storage_backend_impl.cpp b/core/storage/trie/impl/trie_storage_backend_impl.cpp index de76fefa35..cd7c229201 100644 --- a/core/storage/trie/impl/trie_storage_backend_impl.cpp +++ b/core/storage/trie/impl/trie_storage_backend_impl.cpp @@ -8,12 +8,19 @@ #include #include "storage/trie/impl/trie_storage_backend_batch.hpp" +#include "storage/spaces.hpp" namespace kagome::storage::trie { TrieStorageBackendImpl::TrieStorageBackendImpl( - std::shared_ptr storage) - : storage_{std::move(storage)} { + NodeTag, std::shared_ptr storage) + : storage_{storage->getSpace(Space::kTrieNode)} { + BOOST_ASSERT(storage_ != nullptr); + } + + TrieStorageBackendImpl::TrieStorageBackendImpl( + ValueTag, std::shared_ptr storage) + : storage_{storage->getSpace(Space::kTrieValue)} { BOOST_ASSERT(storage_ != nullptr); } diff --git a/core/storage/trie/impl/trie_storage_backend_impl.hpp b/core/storage/trie/impl/trie_storage_backend_impl.hpp index 8a822db432..ddef1444ed 100644 --- a/core/storage/trie/impl/trie_storage_backend_impl.hpp +++ b/core/storage/trie/impl/trie_storage_backend_impl.hpp @@ -8,13 +8,19 @@ #include "common/buffer.hpp" #include "outcome/outcome.hpp" +#include "storage/spaced_storage.hpp" #include "storage/trie/trie_storage_backend.hpp" namespace kagome::storage::trie { - class TrieStorageBackendImpl : public TrieStorageBackend { + class TrieStorageBackendImpl : public TrieNodeStorageBackend, + public TrieValueStorageBackend { public: - TrieStorageBackendImpl(std::shared_ptr storage); + struct NodeTag {}; + TrieStorageBackendImpl(NodeTag, std::shared_ptr storage); + + struct ValueTag {}; + TrieStorageBackendImpl(ValueTag, std::shared_ptr storage); ~TrieStorageBackendImpl() override = default; diff --git a/core/storage/trie/serialization/trie_serializer_impl.cpp b/core/storage/trie/serialization/trie_serializer_impl.cpp index d27155a644..73210a318a 100644 --- a/core/storage/trie/serialization/trie_serializer_impl.cpp +++ b/core/storage/trie/serialization/trie_serializer_impl.cpp @@ -16,13 +16,16 @@ namespace kagome::storage::trie { TrieSerializerImpl::TrieSerializerImpl( std::shared_ptr factory, std::shared_ptr codec, - std::shared_ptr backend) + std::shared_ptr node_backend, + std::shared_ptr value_backend) : trie_factory_{std::move(factory)}, codec_{std::move(codec)}, - backend_{std::move(backend)} { + node_backend_{std::move(node_backend)}, + value_backend_{std::move(value_backend)}{ BOOST_ASSERT(trie_factory_ != nullptr); BOOST_ASSERT(codec_ != nullptr); - BOOST_ASSERT(backend_ != nullptr); + BOOST_ASSERT(node_backend_ != nullptr); + BOOST_ASSERT(value_backend_ != nullptr); } RootHash TrieSerializerImpl::getEmptyRootHash() const { @@ -64,7 +67,7 @@ namespace kagome::storage::trie { outcome::result TrieSerializerImpl::storeRootNode( TrieNode &node, StateVersion version) { - auto batch = backend_->batch(); + auto batch = node_backend_->batch(); BOOST_ASSERT(batch != nullptr); OUTCOME_TRY( @@ -113,7 +116,7 @@ namespace kagome::storage::trie { } Buffer enc; if (db_key.isHash()) { - OUTCOME_TRY(db, backend_->get(db_key.asBuffer())); + OUTCOME_TRY(db, node_backend_->get(db_key.asBuffer())); if (on_node_loaded) { on_node_loaded(db); } @@ -130,7 +133,7 @@ namespace kagome::storage::trie { outcome::result> TrieSerializerImpl::retrieveValue(const common::Hash256 &hash, const OnNodeLoaded &on_node_loaded) const { - OUTCOME_TRY(value, backend_->tryGet(hash)); + OUTCOME_TRY(value, value_backend_->tryGet(hash)); return common::map_optional(std::move(value), [&](common::BufferOrView &&value) { if (on_node_loaded) { diff --git a/core/storage/trie/serialization/trie_serializer_impl.hpp b/core/storage/trie/serialization/trie_serializer_impl.hpp index 6152e46d00..75d9e536d0 100644 --- a/core/storage/trie/serialization/trie_serializer_impl.hpp +++ b/core/storage/trie/serialization/trie_serializer_impl.hpp @@ -13,7 +13,8 @@ namespace kagome::storage::trie { class Codec; class PolkadotTrieFactory; - class TrieStorageBackend; + class TrieNodeStorageBackend; + class TrieValueStorageBackend; struct BranchNode; struct TrieNode; } // namespace kagome::storage::trie @@ -24,7 +25,8 @@ namespace kagome::storage::trie { public: TrieSerializerImpl(std::shared_ptr factory, std::shared_ptr codec, - std::shared_ptr backend); + std::shared_ptr node_backend, + std::shared_ptr value_backend); ~TrieSerializerImpl() override = default; RootHash getEmptyRootHash() const override; @@ -66,7 +68,8 @@ namespace kagome::storage::trie { std::shared_ptr trie_factory_; std::shared_ptr codec_; - std::shared_ptr backend_; + std::shared_ptr node_backend_; + std::shared_ptr value_backend_; }; } // namespace kagome::storage::trie diff --git a/core/storage/trie/trie_storage_backend.hpp b/core/storage/trie/trie_storage_backend.hpp index 4fed148af6..9628d5872d 100644 --- a/core/storage/trie/trie_storage_backend.hpp +++ b/core/storage/trie/trie_storage_backend.hpp @@ -15,12 +15,21 @@ namespace kagome::storage::trie { /** - * Adapter for key-value storages that allows to hide keyspace separation + * Adapter for the trie node storage that allows to hide keyspace separation * along with root hash storing logic from the trie db component */ - class TrieStorageBackend : public BufferStorage { + class TrieNodeStorageBackend : public BufferStorage { public: - ~TrieStorageBackend() override = default; + ~TrieNodeStorageBackend() override = default; + }; + + /** + * Adapter for the trie value storage that allows to hide keyspace separation + * along with root hash storing logic from the trie db component + */ + class TrieValueStorageBackend : public BufferStorage { + public: + ~TrieValueStorageBackend() override = default; }; } // namespace kagome::storage::trie diff --git a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp index cba21c8174..e826db2447 100644 --- a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp +++ b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp @@ -59,20 +59,23 @@ namespace kagome::storage::trie_pruner { TriePrunerImpl::TriePrunerImpl( std::shared_ptr app_state_manager, - std::shared_ptr trie_storage, + std::shared_ptr node_storage, + std::shared_ptr value_storage, std::shared_ptr serializer, std::shared_ptr codec, std::shared_ptr storage, std::shared_ptr hasher, std::shared_ptr config) - : trie_storage_{trie_storage}, + : node_storage_{node_storage}, + value_storage_{value_storage}, serializer_{serializer}, codec_{codec}, storage_{storage}, hasher_{hasher}, pruning_depth_{config->statePruningDepth()}, thorough_pruning_{config->enableThoroughPruning()} { - BOOST_ASSERT(trie_storage_ != nullptr); + BOOST_ASSERT(node_storage_ != nullptr); + BOOST_ASSERT(value_storage_ != nullptr); BOOST_ASSERT(serializer_ != nullptr); BOOST_ASSERT(codec_ != nullptr); BOOST_ASSERT(storage_ != nullptr); @@ -172,9 +175,11 @@ namespace kagome::storage::trie_pruner { outcome::result TriePrunerImpl::pruneFinalized( const primitives::BlockHeader &block) { - auto batch = trie_storage_->batch(); - OUTCOME_TRY(prune(*batch, block.state_root)); - OUTCOME_TRY(batch->commit()); + auto node_batch = node_storage_->batch(); + auto value_batch = value_storage_->batch(); + OUTCOME_TRY(prune(*node_batch, *value_batch, block.state_root)); + OUTCOME_TRY(node_batch->commit()); + OUTCOME_TRY(value_batch->commit()); OUTCOME_TRY(block_enc, scale::encode(block)); auto block_hash = hasher_->blake2b_256(block_enc); @@ -187,13 +192,16 @@ namespace kagome::storage::trie_pruner { outcome::result TriePrunerImpl::pruneDiscarded( const primitives::BlockHeader &block) { // should prune even when pruning depth is none - auto batch = trie_storage_->batch(); - OUTCOME_TRY(prune(*batch, block.state_root)); - OUTCOME_TRY(batch->commit()); + auto node_batch = node_storage_->batch(); + auto value_batch = node_storage_->batch(); + OUTCOME_TRY(prune(*node_batch, *value_batch, block.state_root)); + OUTCOME_TRY(node_batch->commit()); + OUTCOME_TRY(value_batch->commit()); return outcome::success(); } - outcome::result TriePrunerImpl::prune(BufferBatch &batch, + outcome::result TriePrunerImpl::prune(BufferBatch &node_batch, + BufferBatch &value_batch, const trie::RootHash &root_hash) { auto trie_res = serializer_->retrieveTrie(root_hash, nullptr); if (trie_res.has_error() @@ -212,12 +220,12 @@ namespace kagome::storage::trie_pruner { return outcome::success(); } - OUTCOME_TRY( - forEachChildTrie(*trie, - [this, &batch](common::BufferView child_key, - const trie::RootHash &child_hash) { - return prune(batch, child_hash); - })); + OUTCOME_TRY(forEachChildTrie( + *trie, + [this, &node_batch, &value_batch](common::BufferView child_key, + const trie::RootHash &child_hash) { + return prune(node_batch, value_batch, child_hash); + })); size_t nodes_removed = 0; size_t values_removed = 0; @@ -261,17 +269,18 @@ namespace kagome::storage::trie_pruner { if (ref_count == 0) { nodes_removed++; ref_count_.erase(ref_count_it); - OUTCOME_TRY(batch.remove(hash)); + OUTCOME_TRY(node_batch.remove(hash)); auto hash_opt = node->getValue().hash; if (hash_opt.has_value()) { - auto value_ref_it = value_ref_count_.find(hash); + auto &value_hash = *hash_opt; + auto value_ref_it = value_ref_count_.find(value_hash); if (value_ref_it == value_ref_count_.end()) { values_unknown++; } else { auto &value_ref_count = value_ref_it->second; value_ref_count--; if (value_ref_count == 0) { - OUTCOME_TRY(batch.remove(hash)); + OUTCOME_TRY(value_batch.remove(value_hash)); value_ref_count_.erase(value_ref_it); values_removed++; } @@ -374,7 +383,7 @@ namespace kagome::storage::trie_pruner { queued_nodes.pop_back(); auto &ref_count = ref_count_[hash]; if (ref_count == 0 && !thorough_pruning_) { - OUTCOME_TRY(hash_is_in_storage, trie_storage_->contains(hash)); + OUTCOME_TRY(hash_is_in_storage, node_storage_->contains(hash)); if (hash_is_in_storage) { // the node is present in storage but pruner has not indexed it // because pruner has been initialized on a newer state @@ -393,7 +402,12 @@ namespace kagome::storage::trie_pruner { if (is_new_node_with_value) { auto value_hash_opt = encoder.getValueHash(*node, version); if (value_hash_opt) { - value_ref_count_[*value_hash_opt]++; + auto &value_ref_count = value_ref_count_[*value_hash_opt]; + if (value_ref_count == 0 && value_storage_->contains(*value_hash_opt) + && !thorough_pruning_) { + value_ref_count++; + } + value_ref_count++; referenced_values_num++; } } diff --git a/core/storage/trie_pruner/impl/trie_pruner_impl.hpp b/core/storage/trie_pruner/impl/trie_pruner_impl.hpp index 0fabbfec80..a0c3ac56c8 100644 --- a/core/storage/trie_pruner/impl/trie_pruner_impl.hpp +++ b/core/storage/trie_pruner/impl/trie_pruner_impl.hpp @@ -41,7 +41,8 @@ namespace kagome::storage { } namespace kagome::storage::trie { - class TrieStorageBackend; + class TrieNodeStorageBackend; + class TrieValueStorageBackend; class TrieSerializer; class Codec; } // namespace kagome::storage::trie @@ -67,7 +68,8 @@ namespace kagome::storage::trie_pruner { TriePrunerImpl( std::shared_ptr app_state_manager, - std::shared_ptr trie_storage, + std::shared_ptr node_storage, + std::shared_ptr value_storage, std::shared_ptr serializer, std::shared_ptr codec, std::shared_ptr storage, @@ -122,7 +124,8 @@ namespace kagome::storage::trie_pruner { const primitives::BlockHeader &last_pruned_block, const blockchain::BlockTree &block_tree); - outcome::result prune(BufferBatch &batch, + outcome::result prune(BufferBatch &node_batch, + BufferBatch &value_batch, const storage::trie::RootHash &state); outcome::result addNewStateWith( @@ -136,7 +139,8 @@ namespace kagome::storage::trie_pruner { std::unordered_map value_ref_count_; std::optional last_pruned_block_; - std::shared_ptr trie_storage_; + std::shared_ptr node_storage_; + std::shared_ptr value_storage_; std::shared_ptr serializer_; std::shared_ptr codec_; std::shared_ptr storage_; diff --git a/core/utils/kagome_db_editor.cpp b/core/utils/kagome_db_editor.cpp index 778e972b56..d3fddaa849 100644 --- a/core/utils/kagome_db_editor.cpp +++ b/core/utils/kagome_db_editor.cpp @@ -41,8 +41,8 @@ using namespace storage::trie; using common::BufferOrView; using common::BufferView; -struct TrieTracker : TrieStorageBackend { - TrieTracker(std::shared_ptr inner) +struct TrieTracker : TrieNodeStorageBackend, TrieValueStorageBackend { + TrieTracker(std::shared_ptr inner) : inner{std::move(inner)} {} std::unique_ptr cursor() override { @@ -82,7 +82,7 @@ struct TrieTracker : TrieStorageBackend { return keys.count(common::Hash256::fromSpan(key).value()); } - std::shared_ptr inner; + std::shared_ptr inner; mutable std::set keys; }; @@ -246,18 +246,23 @@ int db_editor_main(int argc, const char **argv) { return 0; } - auto trie_buffer_storage = storage->getSpace(storage::Space::kTrieNode); - auto trie_tracker = std::make_shared( - std::make_shared(trie_buffer_storage)); + auto trie_node_tracker = + std::make_shared(std::make_shared( + TrieStorageBackendImpl::NodeTag{}, storage)); + auto trie_value_tracker = + std::make_shared(std::make_shared( + TrieStorageBackendImpl::ValueTag{}, storage)); auto injector = di::make_injector( di::bind.template to([](const auto &injector) { return std::make_shared( injector.template create>(), injector.template create>(), - injector.template create>()); + injector.template create>(), + injector.template create>()); }), - di::bind.template to(trie_tracker), + di::bind.template to(trie_node_tracker), + di::bind.template to(trie_value_tracker), di::bind.template to( std::shared_ptr(nullptr)), di::bind.template to(), @@ -391,34 +396,42 @@ int db_editor_main(int argc, const char **argv) { } } - auto db_cursor = trie_buffer_storage->cursor(); - auto db_batch = trie_buffer_storage->batch(); - auto res = check(db_cursor->seekFirst()); - int count = 0; - { - TicToc t2("Process DB.", log); - while (db_cursor->isValid() && db_cursor->key().has_value()) { - auto key = db_cursor->key().value(); - if (trie_tracker->tracked(key)) { - db_cursor->next().value(); - continue; - } - auto res2 = check(db_batch->remove(key)); - count++; - if (not(count % 10000000)) { - log->trace("{} keys were processed at the db.", count); - res2 = check(db_batch->commit()); - dynamic_cast(buffer_storage.get()) - ->compact(prefix, check(db_cursor->key()).value()); - db_cursor = buffer_storage->cursor(); - db_batch = buffer_storage->batch(); - res = check(db_cursor->seek(key)); + auto trie_node_storage = storage->getSpace(storage::Space::kTrieNode); + auto trie_value_storage = storage->getSpace(storage::Space::kTrieValue); + + auto track_trie_entries = [&log, &buffer_storage, &prefix](auto storage, + auto tracker) { + auto db_cursor = storage->cursor(); + auto db_batch = storage->batch(); + auto res = check(db_cursor->seekFirst()); + int count = 0; + { + TicToc t2("Process DB.", log); + while (db_cursor->isValid() && db_cursor->key().has_value()) { + auto key = db_cursor->key().value(); + if (tracker->tracked(key)) { + db_cursor->next().value(); + continue; + } + auto res2 = check(db_batch->remove(key)); + count++; + if (not(count % 10000000)) { + log->trace("{} keys were processed at the db.", count); + res2 = check(db_batch->commit()); + dynamic_cast(buffer_storage.get()) + ->compact(prefix, check(db_cursor->key()).value()); + db_cursor = buffer_storage->cursor(); + db_batch = buffer_storage->batch(); + res = check(db_cursor->seek(key)); + } + res2 = check(db_cursor->next()); } - res2 = check(db_cursor->next()); + std::ignore = check(db_batch->commit()); } - std::ignore = check(db_batch->commit()); - } - log->trace("{} keys were processed at the db.", ++count); + log->trace("{} keys were processed at the db.", ++count); + }; + track_trie_entries(trie_node_storage, trie_node_tracker); + track_trie_entries(trie_value_storage, trie_value_tracker); { TicToc t4("Compaction 1.", log); From 9c9ba57548a2c37c981eea24e43694cf4bfe653d Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 28 Aug 2023 20:59:03 +0300 Subject: [PATCH 19/43] Fix tests --- test/core/blockchain/block_storage_test.cpp | 2 +- .../authority/authority_manager_test.cpp | 2 +- .../network/state_protocol_observer_test.cpp | 18 +++-- test/core/network/synchronizer_test.cpp | 2 +- .../runtime/trie_storage_provider_test.cpp | 15 ++-- .../changes_trie/changes_tracker_test.cpp | 13 ++- .../trie/trie_storage/trie_batch_test.cpp | 20 ++++- .../trie_storage_backend_test.cpp | 14 +++- .../trie/trie_storage/trie_storage_test.cpp | 10 ++- .../storage/trie_pruner/trie_pruner_test.cpp | 79 +++++++++++-------- test/external-project-test/src/main.cpp | 17 ++-- .../application/app_configuration_mock.hpp | 2 + ..._map_mock.hpp => generic_storage_mock.hpp} | 6 +- .../trie/trie_storage_backend_mock.hpp | 3 +- 14 files changed, 134 insertions(+), 69 deletions(-) rename test/mock/core/storage/{persistent_map_mock.hpp => generic_storage_mock.hpp} (93%) diff --git a/test/core/blockchain/block_storage_test.cpp b/test/core/blockchain/block_storage_test.cpp index 09d8291a43..f74d1c4a72 100644 --- a/test/core/blockchain/block_storage_test.cpp +++ b/test/core/blockchain/block_storage_test.cpp @@ -9,7 +9,7 @@ #include "blockchain/block_storage_error.hpp" #include "mock/core/crypto/hasher_mock.hpp" -#include "mock/core/storage/persistent_map_mock.hpp" +#include "mock/core/storage/generic_storage_mock.hpp" #include "mock/core/storage/spaced_storage_mock.hpp" #include "scale/scale.hpp" #include "storage/database_error.hpp" diff --git a/test/core/consensus/authority/authority_manager_test.cpp b/test/core/consensus/authority/authority_manager_test.cpp index 8061c2120a..718fc4039d 100644 --- a/test/core/consensus/authority/authority_manager_test.cpp +++ b/test/core/consensus/authority/authority_manager_test.cpp @@ -13,7 +13,7 @@ #include "mock/core/blockchain/block_tree_mock.hpp" #include "mock/core/crypto/hasher_mock.hpp" #include "mock/core/runtime/grandpa_api_mock.hpp" -#include "mock/core/storage/persistent_map_mock.hpp" +#include "mock/core/storage/generic_storage_mock.hpp" #include "mock/core/storage/spaced_storage_mock.hpp" #include "primitives/digest.hpp" #include "runtime/executor.hpp" diff --git a/test/core/network/state_protocol_observer_test.cpp b/test/core/network/state_protocol_observer_test.cpp index ee25df89cc..55676f0141 100644 --- a/test/core/network/state_protocol_observer_test.cpp +++ b/test/core/network/state_protocol_observer_test.cpp @@ -11,7 +11,7 @@ #include "mock/core/blockchain/block_header_repository_mock.hpp" #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" #include "network/types/state_request.hpp" -#include "storage/in_memory/in_memory_storage.hpp" +#include "storage/in_memory/in_memory_spaced_storage.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" #include "storage/trie/impl/trie_storage_impl.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_factory_impl.hpp" @@ -38,14 +38,22 @@ using testing::_; using testing::Return; std::shared_ptr makeEmptyInMemoryTrie() { - auto backend = + auto spaced_storage = + std::make_shared(); + auto node_backend = std::make_shared( - std::make_shared()); + kagome::storage::trie::TrieStorageBackendImpl::NodeTag{}, + spaced_storage); + + auto value_backend = + std::make_shared( + kagome::storage::trie::TrieStorageBackendImpl::ValueTag{}, + spaced_storage); auto trie_factory = std::make_shared(); auto codec = std::make_shared(); - auto serializer = - std::make_shared(trie_factory, codec, backend); + auto serializer = std::make_shared( + trie_factory, codec, node_backend, value_backend); auto state_pruner = std::make_shared(); ON_CALL(*state_pruner, addNewState(testing::A(), _)) diff --git a/test/core/network/synchronizer_test.cpp b/test/core/network/synchronizer_test.cpp index 175ae7d0cc..c94645327a 100644 --- a/test/core/network/synchronizer_test.cpp +++ b/test/core/network/synchronizer_test.cpp @@ -21,7 +21,7 @@ #include "mock/core/network/router_mock.hpp" #include "mock/core/runtime/core_mock.hpp" #include "mock/core/runtime/module_factory_mock.hpp" -#include "mock/core/storage/persistent_map_mock.hpp" +#include "mock/core/storage/generic_storage_mock.hpp" #include "mock/core/storage/spaced_storage_mock.hpp" #include "mock/core/storage/trie/serialization/trie_serializer_mock.hpp" #include "mock/core/storage/trie/trie_storage_mock.hpp" diff --git a/test/core/runtime/trie_storage_provider_test.cpp b/test/core/runtime/trie_storage_provider_test.cpp index 78482289a0..711906e9ba 100644 --- a/test/core/runtime/trie_storage_provider_test.cpp +++ b/test/core/runtime/trie_storage_provider_test.cpp @@ -10,7 +10,7 @@ #include "common/buffer.hpp" #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" #include "runtime/common/runtime_transaction_error.hpp" -#include "storage/in_memory/in_memory_storage.hpp" +#include "storage/in_memory/in_memory_spaced_storage.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" #include "storage/trie/impl/trie_storage_impl.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_factory_impl.hpp" @@ -35,15 +35,20 @@ class TrieStorageProviderTest : public ::testing::Test { auto codec = std::make_shared(); - storage_ = std::make_shared(); + storage_ = std::make_shared(); - auto backend = + auto node_backend = std::make_shared( + kagome::storage::trie::TrieStorageBackendImpl::NodeTag{}, storage_); + + auto value_backend = + std::make_shared( + kagome::storage::trie::TrieStorageBackendImpl::ValueTag{}, storage_); auto serializer = std::make_shared( - trie_factory, codec, backend); + trie_factory, codec, node_backend, value_backend); auto state_pruner = std::make_shared(); @@ -61,7 +66,7 @@ class TrieStorageProviderTest : public ::testing::Test { } protected: - std::shared_ptr storage_; + std::shared_ptr storage_; std::shared_ptr storage_provider_; }; diff --git a/test/core/storage/changes_trie/changes_tracker_test.cpp b/test/core/storage/changes_trie/changes_tracker_test.cpp index fd52a0dff3..8aa7fa2f5d 100644 --- a/test/core/storage/changes_trie/changes_tracker_test.cpp +++ b/test/core/storage/changes_trie/changes_tracker_test.cpp @@ -11,6 +11,7 @@ #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" #include "primitives/event_types.hpp" #include "scale/scale.hpp" +#include "storage/in_memory/in_memory_spaced_storage.hpp" #include "storage/in_memory/in_memory_storage.hpp" #include "storage/trie/impl/persistent_trie_batch_impl.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" @@ -28,6 +29,7 @@ using kagome::primitives::BlockHash; using kagome::primitives::ExtrinsicIndex; using kagome::primitives::events::ChainSubscriptionEngine; using kagome::primitives::events::StorageSubscriptionEngine; +using kagome::storage::InMemorySpacedStorage; using kagome::storage::InMemoryStorage; using kagome::storage::changes_trie::ChangesTracker; using kagome::storage::changes_trie::StorageChangesTrackerImpl; @@ -52,10 +54,13 @@ TEST(ChangesTrieTest, IntegrationWithOverlay) { // GIVEN auto factory = std::make_shared(); auto codec = std::make_shared(); - auto backend = std::make_shared( - std::make_shared()); - auto serializer = - std::make_shared(factory, codec, backend); + auto in_memory_storage = std::make_shared(); + auto node_backend = std::make_shared( + TrieStorageBackendImpl::NodeTag{}, in_memory_storage); + auto value_backend = std::make_shared( + TrieStorageBackendImpl::ValueTag{}, in_memory_storage); + auto serializer = std::make_shared( + factory, codec, node_backend, value_backend); auto storage_subscription_engine = std::make_shared(); auto chain_subscription_engine = std::make_shared(); diff --git a/test/core/storage/trie/trie_storage/trie_batch_test.cpp b/test/core/storage/trie/trie_storage/trie_batch_test.cpp index 0a827b80e6..2e30f2a5d4 100644 --- a/test/core/storage/trie/trie_storage/trie_batch_test.cpp +++ b/test/core/storage/trie/trie_storage/trie_batch_test.cpp @@ -6,6 +6,7 @@ #include #include +#include "mock/core/storage/spaced_storage_mock.hpp" #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" #include "storage/changes_trie/impl/storage_changes_tracker_impl.hpp" #include "storage/in_memory/in_memory_storage.hpp" @@ -28,6 +29,7 @@ using kagome::common::BufferView; using kagome::common::Hash256; using kagome::primitives::BlockHash; using kagome::storage::Space; +using kagome::storage::SpacedStorageMock; using kagome::storage::trie::StateVersion; using kagome::storage::trie_pruner::TriePrunerMock; using kagome::subscription::SubscriptionEngine; @@ -50,7 +52,10 @@ class TrieBatchTest : public test::BaseRocksDB_Test { auto serializer = std::make_shared( factory, codec, - std::make_shared(std::move(db_))); + std::make_shared( + TrieStorageBackendImpl::NodeTag{}, rocks_), + std::make_shared( + TrieStorageBackendImpl::ValueTag{}, rocks_)); empty_hash = serializer->getEmptyRootHash(); @@ -188,7 +193,7 @@ TEST_F(TrieBatchTest, Replace) { * consistency */ TEST_F(TrieBatchTest, ConsistentOnFailure) { - auto db = std::make_unique(); + auto db = std::make_shared(); /** * First time the storage will function correctly, after which it will yield * an error @@ -201,10 +206,19 @@ TEST_F(TrieBatchTest, ConsistentOnFailure) { .After(expectation) .WillOnce(Return(PolkadotCodec::Error::UNKNOWN_NODE_TYPE)); + auto spaced_db = std::make_shared(); + ON_CALL(*spaced_db, getSpace(Space::kTrieNode)).WillByDefault(Return(db)); + ON_CALL(*spaced_db, getSpace(Space::kTrieValue)).WillByDefault(Return(db)); + auto factory = std::make_shared(); auto codec = std::make_shared(); auto serializer = std::make_shared( - factory, codec, std::make_shared(std::move(db))); + factory, + codec, + std::make_shared( + TrieStorageBackendImpl::NodeTag{}, spaced_db), + std::make_shared( + TrieStorageBackendImpl::ValueTag{}, spaced_db)); auto state_pruner = std::make_shared(); ON_CALL( *state_pruner, diff --git a/test/core/storage/trie/trie_storage/trie_storage_backend_test.cpp b/test/core/storage/trie/trie_storage/trie_storage_backend_test.cpp index 0fb39a02df..d465d852ff 100644 --- a/test/core/storage/trie/trie_storage/trie_storage_backend_test.cpp +++ b/test/core/storage/trie/trie_storage/trie_storage_backend_test.cpp @@ -7,7 +7,8 @@ #include -#include "mock/core/storage/persistent_map_mock.hpp" +#include "mock/core/storage/generic_storage_mock.hpp" +#include "mock/core/storage/spaced_storage_mock.hpp" #include "mock/core/storage/write_batch_mock.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" #include "testutil/literals.hpp" @@ -16,6 +17,7 @@ using kagome::common::Buffer; using kagome::common::BufferView; using kagome::storage::BufferStorageMock; +using kagome::storage::SpacedStorageMock; using kagome::storage::face::WriteBatchMock; using kagome::storage::trie::TrieStorageBackendImpl; using testing::Invoke; @@ -23,9 +25,17 @@ using testing::Return; class TrieDbBackendTest : public testing::Test { public: + void SetUp() override { + ON_CALL(*spaced_storage, getSpace(kagome::storage::Space::kTrieNode)) + .WillByDefault(Return(storage)); + } + std::shared_ptr storage = std::make_shared(); - TrieStorageBackendImpl backend{storage}; + std::shared_ptr spaced_storage = + std::make_shared(); + TrieStorageBackendImpl backend{TrieStorageBackendImpl::NodeTag{}, + spaced_storage}; }; /** diff --git a/test/core/storage/trie/trie_storage/trie_storage_test.cpp b/test/core/storage/trie/trie_storage/trie_storage_test.cpp index 0d5a9ab247..6363ffd5a0 100644 --- a/test/core/storage/trie/trie_storage/trie_storage_test.cpp +++ b/test/core/storage/trie/trie_storage/trie_storage_test.cpp @@ -59,7 +59,9 @@ TEST(TriePersistencyTest, CreateDestroyCreate) { factory, codec, std::make_shared( - rocks_db->getSpace(Space::kDefault))); + TrieStorageBackendImpl::NodeTag{}, rocks_db), + std::make_shared( + TrieStorageBackendImpl::ValueTag{}, rocks_db)); auto state_pruner = std::make_shared(); ON_CALL(*state_pruner, @@ -87,7 +89,11 @@ TEST(TriePersistencyTest, CreateDestroyCreate) { factory, codec, std::make_shared( - new_rocks_db->getSpace(Space::kDefault))); + TrieStorageBackendImpl::NodeTag{}, + new_rocks_db), + std::make_shared( + TrieStorageBackendImpl::ValueTag{}, + new_rocks_db)); auto state_pruner = std::make_shared(); auto storage = TrieStorageImpl::createFromStorage(codec, serializer, state_pruner) diff --git a/test/core/storage/trie_pruner/trie_pruner_test.cpp b/test/core/storage/trie_pruner/trie_pruner_test.cpp index 23766dd33f..177d9a9516 100644 --- a/test/core/storage/trie_pruner/trie_pruner_test.cpp +++ b/test/core/storage/trie_pruner/trie_pruner_test.cpp @@ -12,7 +12,7 @@ #include "mock/core/application/app_configuration_mock.hpp" #include "mock/core/application/app_state_manager_mock.hpp" #include "mock/core/blockchain/block_tree_mock.hpp" -#include "mock/core/storage/persistent_map_mock.hpp" +#include "mock/core/storage/generic_storage_mock.hpp" #include "mock/core/storage/spaced_storage_mock.hpp" #include "mock/core/storage/trie/polkadot_trie_cursor_mock.h" #include "mock/core/storage/trie/serialization/codec_mock.hpp" @@ -166,7 +166,9 @@ class TriePrunerTest : public testing::Test { std::make_shared(); ON_CALL(*config_mock, statePruningDepth()).WillByDefault(Return(16)); - trie_storage_mock.reset( + trie_node_storage_mock.reset( + new testing::NiceMock()); + trie_value_storage_mock.reset( new testing::NiceMock()); persistent_storage_mock.reset( new testing::NiceMock); @@ -189,7 +191,8 @@ class TriePrunerTest : public testing::Test { pruner.reset(new TriePrunerImpl( std::make_shared(), - trie_storage_mock, + trie_node_storage_mock, + trie_value_storage_mock, serializer_mock, codec_mock, persistent_storage_mock, @@ -214,7 +217,8 @@ class TriePrunerTest : public testing::Test { pruner.reset(new TriePrunerImpl( std::make_shared(), - trie_storage_mock, + trie_node_storage_mock, + trie_value_storage_mock, serializer_mock, codec_mock, persistent_storage_mock, @@ -259,7 +263,8 @@ class TriePrunerTest : public testing::Test { std::unique_ptr pruner; std::shared_ptr serializer_mock; - std::shared_ptr trie_storage_mock; + std::shared_ptr trie_node_storage_mock; + std::shared_ptr trie_value_storage_mock; std::shared_ptr> persistent_storage_mock; std::shared_ptr codec_mock; std::shared_ptr hasher; @@ -349,7 +354,7 @@ TEST_F(TriePrunerTest, BasicScenario) { {{"_0"_hash256, makeTransparentNode({NODE, "_0"_hash256, {}})}, {"_5"_hash256, makeTransparentNode({NODE, "_5"_hash256, {}})}}})); - EXPECT_CALL(*trie_storage_mock, batch()).WillRepeatedly(Invoke([]() { + EXPECT_CALL(*trie_node_storage_mock, batch()).WillRepeatedly(Invoke([]() { auto batch = std::make_unique>(); EXPECT_CALL(*batch, remove(_)).WillRepeatedly(Return(outcome::success())); EXPECT_CALL(*batch, commit()).WillOnce(Return(outcome::success())); @@ -468,7 +473,7 @@ TEST_F(TriePrunerTest, RandomTree) { std::map node_storage; std::set inserted_keys; - EXPECT_CALL(*trie_storage_mock, get(_)) + EXPECT_CALL(*trie_node_storage_mock, get(_)) .WillRepeatedly( Invoke([&node_storage](auto &k) -> outcome::result { auto it = node_storage.find(k); @@ -478,7 +483,8 @@ TEST_F(TriePrunerTest, RandomTree) { return BufferOrView{it->second.view()}; })); - trie::TrieSerializerImpl serializer{trie_factory, codec, trie_storage_mock}; + trie::TrieSerializerImpl serializer{ + trie_factory, codec, trie_node_storage_mock, trie_value_storage_mock}; std::vector> kv; std::mt19937 rand; rand.seed(42); @@ -499,18 +505,19 @@ TEST_F(TriePrunerTest, RandomTree) { })); for (unsigned i = 0; i < STATES_NUM; i++) { - EXPECT_CALL(*trie_storage_mock, batch()).WillOnce(Invoke([&node_storage]() { - auto batch_mock = - std::make_unique>(); - EXPECT_CALL(*batch_mock, put(_, _)) - .WillRepeatedly(Invoke([&node_storage](auto &k, auto &v) { - node_storage[k] = v; - return outcome::success(); - })); - EXPECT_CALL(*batch_mock, commit()) - .WillRepeatedly(Return(outcome::success())); - return batch_mock; - })); + EXPECT_CALL(*trie_node_storage_mock, batch()) + .WillOnce(Invoke([&node_storage]() { + auto batch_mock = + std::make_unique>(); + EXPECT_CALL(*batch_mock, put(_, _)) + .WillRepeatedly(Invoke([&node_storage](auto &k, auto &v) { + node_storage[k] = v; + return outcome::success(); + })); + EXPECT_CALL(*batch_mock, commit()) + .WillRepeatedly(Return(outcome::success())); + return batch_mock; + })); for (unsigned j = 0; j < INSERT_PER_STATE; j++) { auto k = randomBuffer(rand); @@ -548,7 +555,7 @@ TEST_F(TriePrunerTest, RandomTree) { roots.push_back(root); if (i >= 16) { - EXPECT_CALL(*trie_storage_mock, batch()) + EXPECT_CALL(*trie_node_storage_mock, batch()) .WillOnce(Invoke([&node_storage]() { auto batch = std::make_unique>(); @@ -568,16 +575,17 @@ TEST_F(TriePrunerTest, RandomTree) { } } for (unsigned i = STATES_NUM - 16; i < STATES_NUM; i++) { - EXPECT_CALL(*trie_storage_mock, batch()).WillOnce(Invoke([&node_storage]() { - auto batch = std::make_unique>(); - EXPECT_CALL(*batch, remove(_)) - .WillRepeatedly(Invoke([&node_storage](auto &k) { - node_storage.erase(k); - return outcome::success(); - })); - EXPECT_CALL(*batch, commit()).WillOnce(Return(outcome::success())); - return batch; - })); + EXPECT_CALL(*trie_node_storage_mock, batch()) + .WillOnce(Invoke([&node_storage]() { + auto batch = std::make_unique>(); + EXPECT_CALL(*batch, remove(_)) + .WillRepeatedly(Invoke([&node_storage](auto &k) { + node_storage.erase(k); + return outcome::success(); + })); + EXPECT_CALL(*batch, commit()).WillOnce(Return(outcome::success())); + return batch; + })); auto &root = roots[i]; ASSERT_OUTCOME_SUCCESS_TRY( @@ -624,7 +632,7 @@ TEST_F(TriePrunerTest, RestoreStateFromGenesis) { ON_CALL(*block_tree, getChildren(_)) .WillByDefault(Return(std::vector{})); - + ON_CALL(*block_tree, bestLeaf()) .WillByDefault(Return(BlockInfo{6, hash_from_header(headers.at(6))})); @@ -695,7 +703,7 @@ TEST_F(TriePrunerTest, FastSyncScenario) { auto block_tree = std::make_shared>(); - ON_CALL(*trie_storage_mock, get(_)) + ON_CALL(*trie_node_storage_mock, get(_)) .WillByDefault(Invoke( [&](auto &key) -> outcome::result { if (node_storage.count(key) == 0) { @@ -704,7 +712,7 @@ TEST_F(TriePrunerTest, FastSyncScenario) { return kagome::common::BufferOrView{node_storage.at(key).view()}; })); - ON_CALL(*trie_storage_mock, batch()).WillByDefault(Invoke([&]() { + ON_CALL(*trie_node_storage_mock, batch()).WillByDefault(Invoke([&]() { auto batch = std::make_unique< testing::NiceMock>>(); ON_CALL(*batch, put(_, _)) @@ -732,7 +740,8 @@ TEST_F(TriePrunerTest, FastSyncScenario) { codec->encodeNode(*genesis_trie->getRoot(), trie::StateVersion::V0) .value()); - trie::TrieSerializerImpl serializer{trie_factory, codec, trie_storage_mock}; + trie::TrieSerializerImpl serializer{ + trie_factory, codec, trie_node_storage_mock, trie_value_storage_mock}; ON_CALL(*serializer_mock, retrieveTrie(genesis_state_root, _)) .WillByDefault(Return(genesis_trie)); diff --git a/test/external-project-test/src/main.cpp b/test/external-project-test/src/main.cpp index a3c107ea13..b6a168670f 100644 --- a/test/external-project-test/src/main.cpp +++ b/test/external-project-test/src/main.cpp @@ -95,7 +95,6 @@ int main() { .value(); auto code_substitutes = chain_spec->codeSubstitutes(); - auto storage = std::make_shared(); auto config = std::make_shared( kagome::log::createLogger("AppConfiguration")); @@ -103,10 +102,15 @@ int main() { auto trie_factory = std::make_shared(); auto codec = std::make_shared(); - auto storage_backend = - std::make_shared(storage); + auto node_storage_backend = + std::make_shared( + kagome::storage::trie::TrieStorageBackendImpl::NodeTag{}, database); + auto value_storage_backend = + std::make_shared( + kagome::storage::trie::TrieStorageBackendImpl::ValueTag{}, + database); auto serializer = std::make_shared( - trie_factory, codec, storage_backend); + trie_factory, codec, node_storage_backend, value_storage_backend); auto app_state_manager = std::make_shared(); @@ -114,7 +118,8 @@ int main() { auto state_pruner = std::make_shared( app_state_manager, - storage_backend, + node_storage_backend, + value_storage_backend, serializer, codec, database, @@ -242,7 +247,7 @@ int main() { [[maybe_unused]] auto ctx_factory = std::make_shared(module_repo, - header_repo); + header_repo); [[maybe_unused]] auto executor = kagome::runtime::Executor(ctx_factory, cache); diff --git a/test/mock/core/application/app_configuration_mock.hpp b/test/mock/core/application/app_configuration_mock.hpp index c4cbd61665..91365933a7 100644 --- a/test/mock/core/application/app_configuration_mock.hpp +++ b/test/mock/core/application/app_configuration_mock.hpp @@ -156,6 +156,8 @@ namespace kagome::application { MOCK_METHOD(bool, shouldPruneDiscardedStates, (), (const, override)); + MOCK_METHOD(bool, enableThoroughPruning, (), (const, override)); + MOCK_METHOD(StorageBackend, storageBackend, (), (const, override)); MOCK_METHOD(uint32_t, dbCacheSize, (), (const, override)); diff --git a/test/mock/core/storage/persistent_map_mock.hpp b/test/mock/core/storage/generic_storage_mock.hpp similarity index 93% rename from test/mock/core/storage/persistent_map_mock.hpp rename to test/mock/core/storage/generic_storage_mock.hpp index 9563f34e31..6a8e6ed4a1 100644 --- a/test/mock/core/storage/persistent_map_mock.hpp +++ b/test/mock/core/storage/generic_storage_mock.hpp @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef KAGOME_PERSISTENT_MAP_MOCK_HPP -#define KAGOME_PERSISTENT_MAP_MOCK_HPP +#ifndef KAGOME_GENERIC_STORAGE_MOCK_HPP +#define KAGOME_GENERIC_STORAGE_MOCK_HPP #include @@ -58,4 +58,4 @@ namespace kagome::storage { using BufferStorageMock = face::GenericStorageMock; } // namespace kagome::storage -#endif // KAGOME_PERSISTENT_MAP_MOCK_HPP +#endif // KAGOME_GENERIC_STORAGE_MOCK_HPP diff --git a/test/mock/core/storage/trie/trie_storage_backend_mock.hpp b/test/mock/core/storage/trie/trie_storage_backend_mock.hpp index ec037c70d7..13c6492b5e 100644 --- a/test/mock/core/storage/trie/trie_storage_backend_mock.hpp +++ b/test/mock/core/storage/trie/trie_storage_backend_mock.hpp @@ -12,7 +12,8 @@ namespace kagome::storage::trie { - class TrieStorageBackendMock : public TrieStorageBackend { + class TrieStorageBackendMock : public TrieNodeStorageBackend, + public TrieValueStorageBackend { public: MOCK_METHOD(std::unique_ptr, batch, (), (override)); From dc44749a857603fb9e236c341e6fc4bfdc399d44 Mon Sep 17 00:00:00 2001 From: Harrm Date: Fri, 1 Sep 2023 11:05:45 +0300 Subject: [PATCH 20/43] WasmEdge integration in progress --- cmake/Hunter/hunter-gate-url.cmake | 4 +- cmake/dependencies.cmake | 3 + core/application/app_configuration.hpp | 7 +- .../impl/app_configuration_impl.cpp | 7 +- core/injector/application_injector.cpp | 74 +-- core/runtime/CMakeLists.txt | 1 + core/runtime/types.hpp | 13 +- core/runtime/wasm_edge/CMakeLists.txt | 4 + .../wasm_edge/core_api_factory_impl.cpp | 5 + .../wasm_edge/core_api_factory_impl.hpp | 16 + .../runtime/wasm_edge/module_factory_impl.cpp | 478 ++++++++++++++++++ .../runtime/wasm_edge/module_factory_impl.hpp | 22 + 12 files changed, 590 insertions(+), 44 deletions(-) create mode 100644 core/runtime/wasm_edge/CMakeLists.txt create mode 100644 core/runtime/wasm_edge/core_api_factory_impl.cpp create mode 100644 core/runtime/wasm_edge/core_api_factory_impl.hpp create mode 100644 core/runtime/wasm_edge/module_factory_impl.cpp create mode 100644 core/runtime/wasm_edge/module_factory_impl.hpp diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake index c4d2d2a594..4a6b44eb0f 100644 --- a/cmake/Hunter/hunter-gate-url.cmake +++ b/cmake/Hunter/hunter-gate-url.cmake @@ -1,5 +1,5 @@ HunterGate( - URL "https://github.com/soramitsu/soramitsu-hunter/archive/refs/tags/v0.23.257-soramitsu58.zip" - SHA1 "0a8363ab7833f0d34a0bf15e3b767eab9d598474" + URL "https://github.com/soramitsu/soramitsu-hunter/archive/heads/feature/wasm-edge.zip" + SHA1 "9af766bdf351afdd58cda26142c2c42c451b5e9f" LOCAL ) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index cf633e2494..5f37f7ec41 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -79,6 +79,9 @@ hunter_add_package(wavm) find_package(LLVM CONFIG REQUIRED) find_package(WAVM CONFIG REQUIRED) +hunter_add_package(WasmEdge) +find_library(WASM_EDGE_LIBRARY NAMES wasmedge REQUIRED PATHS "${WASMEDGE_ROOT}") + hunter_add_package(zstd) find_package(zstd CONFIG REQUIRED) diff --git a/core/application/app_configuration.hpp b/core/application/app_configuration.hpp index 55b88c3869..5b7e9569ce 100644 --- a/core/application/app_configuration.hpp +++ b/core/application/app_configuration.hpp @@ -215,7 +215,12 @@ namespace kagome::application { */ virtual SyncMethod syncMethod() const = 0; - enum class RuntimeExecutionMethod { Compile, Interpret }; + enum class RuntimeExecutionMethod { + CompileWavm, + CompileWasmEdge, + Interpret + }; + /** * @return enum constant of the chosen runtime backend */ diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index f63a2a7813..2cc7c74368 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -159,8 +159,11 @@ namespace { if (str == "Interpreted") { return REM::Interpret; } - if (str == "Compiled") { - return REM::Compile; + if (str == "Compiled-Wavm") { + return REM::CompileWavm; + } + if (str == "Compiled-WasmEdge") { + return REM::CompileWasmEdge; } return std::nullopt; } diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 8b2b9bd388..ddca8ff279 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -147,6 +147,8 @@ #include "runtime/runtime_api/impl/session_keys_api.hpp" #include "runtime/runtime_api/impl/tagged_transaction_queue.hpp" #include "runtime/runtime_api/impl/transaction_payment_api.hpp" +#include "runtime/wasm_edge/module_factory_impl.hpp" +#include "runtime/wasm_edge/core_api_factory_impl.hpp" #include "runtime/wavm/compartment_wrapper.hpp" #include "runtime/wavm/core_api_factory_impl.hpp" #include "runtime/wavm/instance_environment_factory.hpp" @@ -339,9 +341,7 @@ namespace { } template - auto makeWavmInjector( - application::AppConfiguration::RuntimeExecutionMethod method, - Ts &&...args) { + auto makeWavmInjector(Ts &&...args) { return di::make_injector( bind_by_lambda([](const auto &injector) { @@ -365,14 +365,33 @@ namespace { .template create>(); return module->instantiate(); }), + bind_by_lambda([](const auto + &injector) { + std::optional> + module_cache_opt; + auto &app_config = + injector.template create(); + if (app_config.useWavmCache()) { + module_cache_opt = std::make_shared( + injector.template create>(), + app_config.runtimeCacheDirPath()); + } + return std::make_shared( + injector + .template create>(), + injector.template create>(), + injector.template create< + sptr>(), + injector.template create>(), + module_cache_opt, + injector.template create>()); + }), di::bind.template to(), std::forward(args)...); } template - auto makeBinaryenInjector( - application::AppConfiguration::RuntimeExecutionMethod method, - Ts &&...args) { + auto makeBinaryenInjector(Ts &&...args) { return di::make_injector( bind_by_lambda( [](const auto &injector) { @@ -389,9 +408,15 @@ namespace { std::forward(args)...); } + template + auto makeWasmEdgeInjector(Ts &&...args) { + return di::make_injector(std::forward(args)...); + } + template auto choose_runtime_implementation( const Injector &injector, @@ -402,9 +427,12 @@ namespace { case RuntimeExecutionMethod::Interpret: return std::static_pointer_cast( injector.template create>()); - case RuntimeExecutionMethod::Compile: + case RuntimeExecutionMethod::CompileWavm: return std::static_pointer_cast( injector.template create>()); + case RuntimeExecutionMethod::CompileWasmEdge: + return std::static_pointer_cast( + injector.template create>()); } throw std::runtime_error("Unknown runtime execution method"); } @@ -448,8 +476,9 @@ namespace { return injector .template create>(); }), - makeWavmInjector(method), - makeBinaryenInjector(method), + makeWavmInjector(), + makeBinaryenInjector(), + makeWasmEdgeInjector(), bind_by_lambda([](const auto &injector) { return std::make_shared(); }), @@ -458,34 +487,15 @@ namespace { return choose_runtime_implementation< runtime::CoreApiFactory, runtime::binaryen::CoreApiFactoryImpl, - runtime::wavm::CoreApiFactoryImpl>(injector, method); - }), - bind_by_lambda([](const auto - &injector) { - std::optional> - module_cache_opt; - auto &app_config = - injector.template create(); - if (app_config.useWavmCache()) { - module_cache_opt = std::make_shared( - injector.template create>(), - app_config.runtimeCacheDirPath()); - } - return std::make_shared( - injector - .template create>(), - injector.template create>(), - injector.template create< - sptr>(), - injector.template create>(), - module_cache_opt, - injector.template create>()); + runtime::wavm::CoreApiFactoryImpl, + runtime::wasm_edge::CoreApiFactoryImpl>(injector, method); }), bind_by_lambda([method](const auto &injector) { return choose_runtime_implementation< runtime::ModuleFactory, runtime::binaryen::ModuleFactoryImpl, - runtime::wavm::ModuleFactoryImpl>(injector, method); + runtime::wavm::ModuleFactoryImpl, + runtime::wasm_edge::ModuleFactoryImpl>(injector, method); }), di::bind.template to(), di::bind.template to(), diff --git a/core/runtime/CMakeLists.txt b/core/runtime/CMakeLists.txt index a80994f6a9..5458855ab3 100644 --- a/core/runtime/CMakeLists.txt +++ b/core/runtime/CMakeLists.txt @@ -6,5 +6,6 @@ add_subdirectory(common) add_subdirectory(binaryen) add_subdirectory(wavm) +add_subdirectory(wasm_edge) add_subdirectory(runtime_api/impl) diff --git a/core/runtime/types.hpp b/core/runtime/types.hpp index 5dbb068914..6afb4d2053 100644 --- a/core/runtime/types.hpp +++ b/core/runtime/types.hpp @@ -21,23 +21,22 @@ namespace kagome::runtime { Debug = 3, Trace = 4, }; - /** - * @brief type of wasm memory is 32 bit integer - */ + using WasmPointer = uint32_t; + /** * @brief combination of pointer and size, where less significant part * represents wasm pointer, and most significant represents size */ using WasmSpan = uint64_t; + /** - * @brief Size type is uint32_t because we are working in 32 bit address space + * @brief Size type is int32_t because we are working in 32 bit address space */ using WasmSize = uint32_t; - /** - * @brief Enum value is uint32_t - */ + using WasmEnum = uint32_t; + /** * @brief Offset type is uint32_t because we are working in 32 bit address * space diff --git a/core/runtime/wasm_edge/CMakeLists.txt b/core/runtime/wasm_edge/CMakeLists.txt new file mode 100644 index 0000000000..f655b57be4 --- /dev/null +++ b/core/runtime/wasm_edge/CMakeLists.txt @@ -0,0 +1,4 @@ + +add_library(runtime_wasm_edge module_factory_impl.cpp) +target_link_libraries(runtime_wasm_edge "${WASM_EDGE_LIBRARY}") +target_include_directories(runtime_wasm_edge PRIVATE "${WASMEDGE_ROOT}/include") diff --git a/core/runtime/wasm_edge/core_api_factory_impl.cpp b/core/runtime/wasm_edge/core_api_factory_impl.cpp new file mode 100644 index 0000000000..7127d104ea --- /dev/null +++ b/core/runtime/wasm_edge/core_api_factory_impl.cpp @@ -0,0 +1,5 @@ +/** + * Created by harrm on 30.08.23. + */ + +#include "core_api_factory_impl.hpp" diff --git a/core/runtime/wasm_edge/core_api_factory_impl.hpp b/core/runtime/wasm_edge/core_api_factory_impl.hpp new file mode 100644 index 0000000000..cb0698e67b --- /dev/null +++ b/core/runtime/wasm_edge/core_api_factory_impl.hpp @@ -0,0 +1,16 @@ +/** +* Copyright Soramitsu Co., Ltd. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0 +*/ + +#pragma once + +#include "runtime/core_api_factory.hpp" + +namespace kagome::runtime::wasm_edge { + + class CoreApiFactoryImpl final: public CoreApiFactory { + + }; + +} diff --git a/core/runtime/wasm_edge/module_factory_impl.cpp b/core/runtime/wasm_edge/module_factory_impl.cpp new file mode 100644 index 0000000000..7b96961add --- /dev/null +++ b/core/runtime/wasm_edge/module_factory_impl.cpp @@ -0,0 +1,478 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "module_factory_impl.hpp" + +#include + +#include "host_api/host_api.hpp" +#include "runtime/memory_provider.hpp" +#include "runtime/module.hpp" +#include "runtime/module_instance.hpp" + +namespace kagome::runtime::wasm_edge { + enum class Error { + INVALID_VALUE_TYPE = 1, + + }; +} + +OUTCOME_HPP_DECLARE_ERROR(kagome::runtime::wasm_edge, Error); + +OUTCOME_CPP_DEFINE_CATEGORY(kagome::runtime::wasm_edge, Error, e) { + using E = kagome::runtime::wasm_edge::Error; + switch (e) { + case kagome::runtime::wasm_edge::Error::INVALID_VALUE_TYPE: + return "Invalid value type"; + } + return "Unknown WasmEdge error"; +} + +class WasmEdgeErrCategory final : public std::error_category { + public: + const char *name() const noexcept override { + return "WasmEdge"; + } + + std::string message(int code) const override { + auto res = WasmEdge_ResultGen(WasmEdge_ErrCategory_WASM, code); + return WasmEdge_ResultGetMessage(res); + } +}; + +namespace kagome::runtime::wasm_edge { + + template + class Wrapper { + public: + Wrapper() : t{} {} + + Wrapper(T t) : t{std::move(t)} {} + + Wrapper(const Wrapper &) = delete; + Wrapper &operator=(const Wrapper &) = delete; + + Wrapper(Wrapper &&other) { + if (t) { + Deleter(t); + } + t = other.t; + other.t = nullptr; + } + + Wrapper &operator=(Wrapper &&other) { + if (t) { + Deleter(t); + } + t = other.t; + other.t = nullptr; + return *this; + } + + ~Wrapper() { + if constexpr (std::is_pointer_v) { + if (t) { + Deleter(t); + } + } + } + + T &raw() { + return t; + } + + const T &raw() const { + return t; + } + + T *operator->() { + return t; + } + + const T *operator->() const { + return t; + } + + const T *operator&() const { + return &t; + } + + T *operator&() { + return &t; + } + + bool operator==(std::nullptr_t) const { + return t == nullptr; + } + + bool operator!=(std::nullptr_t) const { + return t != nullptr; + } + + T t = nullptr; + }; + + using ConfigureContext = + Wrapper; + using LoaderContext = + Wrapper; + using StatsContext = + Wrapper; + using FunctionTypeContext = + Wrapper; + using FunctionInstanceContext = Wrapper; + using ExecutorContext = + Wrapper; + using StoreContext = Wrapper; + using VmContext = Wrapper; + using ModuleInstanceContext = + Wrapper; + using String = Wrapper; + using ASTModuleContext = + Wrapper; + using MemoryInstanceContext = + Wrapper; + + static outcome::result convertValue(WasmEdge_Value v) { + switch (v.Type) { + case WasmEdge_ValType_I32: + return WasmEdge_ValueGetI32(v); + case WasmEdge_ValType_I64: + return WasmEdge_ValueGetI64(v); + case WasmEdge_ValType_F32: + return WasmEdge_ValueGetF32(v); + case WasmEdge_ValType_F64: + return WasmEdge_ValueGetF64(v); + case WasmEdge_ValType_V128: + return Error::INVALID_VALUE_TYPE; + case WasmEdge_ValType_FuncRef: + return Error::INVALID_VALUE_TYPE; + case WasmEdge_ValType_ExternRef: + return Error::INVALID_VALUE_TYPE; + } + } + + class ModuleInstanceImpl : public ModuleInstance { + public: + explicit ModuleInstanceImpl(std::shared_ptr module, + std::shared_ptr executor, + ModuleInstanceContext instance_ctx, + InstanceEnvironment env, + const common::Hash256 &code_hash) + : module_{module}, + executor_{executor}, + instance_{std::move(instance_ctx)}, + env_{std::move(env)}, + code_hash_{} { + BOOST_ASSERT(module_ != nullptr); + BOOST_ASSERT(executor_ != nullptr); + BOOST_ASSERT(instance_ != nullptr); + } + + const common::Hash256 &getCodeHash() const override { + return code_hash_; + } + + std::shared_ptr getModule() const override { + return module_; + } + + outcome::result callExportFunction( + std::string_view name, common::BufferView encoded_args) const override { + ConfigureContext config_ctx = WasmEdge_ConfigureCreate(); + StoreContext store_ctx = WasmEdge_StoreCreate(); + VmContext vm_ctx = WasmEdge_VMCreate(config_ctx.raw(), store_ctx.raw()); + + std::array params{WasmEdge_ValueGenI32(42), WasmEdge_ValueGenI32(42)}; + std::array returns{WasmEdge_ValueGenI64(42)}; + String wasm_name = + WasmEdge_StringCreateByBuffer(name.data(), name.size()); + auto res = WasmEdge_VMExecute(vm_ctx.raw(), + wasm_name.raw(), + params.data(), + params.size(), + returns.data(), + returns.size()); + if (!WasmEdge_ResultOK(res)) { + return std::error_code{static_cast(WasmEdge_ResultGetCode(res)), + WasmEdgeErrCategory{}}; + } + return PtrSize{static_cast(WasmEdge_ValueGetI64(returns[0]))}; + } + + outcome::result> getGlobal( + std::string_view name_) const override { + String name = WasmEdge_StringCreateByBuffer(name_.data(), name_.size()); + auto global = + WasmEdge_ModuleInstanceFindGlobal(instance_.raw(), name.raw()); + if (global == nullptr) { + return std::nullopt; + } + auto value = WasmEdge_GlobalInstanceGetValue(global); + OUTCOME_TRY(v, convertValue(value)); + return v; + } + + void forDataSegment(const DataSegmentProcessor &callback) const override { +#warning "Implement before PR" + } + + const InstanceEnvironment &getEnvironment() const override { + return env_; + } + + outcome::result resetEnvironment() override { + env_.host_api->reset(); + return outcome::success(); + } + + outcome::result resetMemory(const MemoryLimits &config) override { +#warning "Implement before PR" + return outcome::success(); + } + + private: + std::shared_ptr module_; + std::shared_ptr executor_; + ModuleInstanceContext instance_; + InstanceEnvironment env_; + const common::Hash256 code_hash_; + }; + + class MemoryImpl final : public Memory { + public: + MemoryImpl(MemoryInstanceContext mem_instance, MemoryAllocator allocator) + : mem_instance_{std::move(mem_instance)}, allocator_{std::move(allocator)} { + BOOST_ASSERT(mem_instance_ != nullptr); + } + /** + * @brief Return the size of the memory + */ + WasmSize size() const override { + return WasmEdge_MemoryInstanceGetPageSize(mem_instance_.raw()) + * kMemoryPageSize; + } + + /** + * Resizes memory to the given size + * @param new_size + */ + void resize(WasmSize new_size) override { + if (new_size > size()) { + auto old_page_num = + WasmEdge_MemoryInstanceGetPageSize(mem_instance_.raw()); + auto new_page_num = (new_size + kMemoryPageSize - 1) / kMemoryPageSize; + auto res = WasmEdge_MemoryInstanceGrowPage(mem_instance_.raw(), + new_page_num - old_page_num); + BOOST_ASSERT(WasmEdge_ResultOK(res)); + } + } + + WasmPointer allocate(WasmSize size) override { + return allocator_.allocate(size); + } + + std::optional deallocate(WasmPointer ptr) override { + return allocator_.deallocate(ptr); + } + + template + T loadInt(WasmPointer addr) const { + T data{}; + auto res = + WasmEdge_MemoryInstanceGetData(mem_instance_.raw(), + reinterpret_cast(&data), + addr, + sizeof(T)); + BOOST_ASSERT(WasmEdge_ResultOK(res)); + return data; + } + + int8_t load8s(WasmPointer addr) const override { + return loadInt(addr); + } + + uint8_t load8u(WasmPointer addr) const override { + return loadInt(addr); + } + + int16_t load16s(WasmPointer addr) const override { + return loadInt(addr); + } + + uint16_t load16u(WasmPointer addr) const override { + return loadInt(addr); + } + + int32_t load32s(WasmPointer addr) const override { + return loadInt(addr); + } + + uint32_t load32u(WasmPointer addr) const override { + return loadInt(addr); + } + + int64_t load64s(WasmPointer addr) const override { + return loadInt(addr); + } + + uint64_t load64u(WasmPointer addr) const override { + return loadInt(addr); + } + + std::array load128(WasmPointer addr) const override { + return loadInt>(addr); + } + + common::BufferView loadN(WasmPointer addr, WasmSize n) const override { + auto ptr = + WasmEdge_MemoryInstanceGetPointer(mem_instance_.raw(), addr, n); + BOOST_ASSERT(ptr); + return common::BufferView{ptr, n}; + } + + std::string loadStr(WasmPointer addr, WasmSize n) const override { + std::string res(n, ' '); + auto span = loadN(addr, n); + std::copy_n(span.begin(), n, res.begin()); + return res; + } + + template + void storeInt(WasmPointer addr, T value) const { + auto res = + WasmEdge_MemoryInstanceSetData(mem_instance_.raw(), + reinterpret_cast(&value), + addr, + sizeof(T)); + BOOST_ASSERT(WasmEdge_ResultOK(res)); + } + + void store8(WasmPointer addr, int8_t value) override { + storeInt(addr, value); + } + + void store16(WasmPointer addr, int16_t value) override { + storeInt(addr, value); + } + + void store32(WasmPointer addr, int32_t value) override { + storeInt(addr, value); + } + + void store64(WasmPointer addr, int64_t value) override { + storeInt(addr, value); + } + + void store128(WasmPointer addr, + const std::array &value) override { + storeInt>(addr, value); + } + + void storeBuffer(WasmPointer addr, + gsl::span value) override { + auto res = WasmEdge_MemoryInstanceSetData( + mem_instance_.raw(), value.data(), addr, value.size()); + BOOST_ASSERT(WasmEdge_ResultOK(res)); + } + + /** + * @brief allocates buffer in memory and copies value into memory + * @param value buffer to store + * @return full wasm pointer to allocated buffer + */ + WasmSpan storeBuffer(gsl::span value) override { + auto ptr = allocate(value.size()); + storeBuffer(ptr, value); + return PtrSize{ptr, static_cast(value.size())}.combine(); + } + + private: + MemoryInstanceContext mem_instance_; + MemoryAllocator allocator_; + }; + + class MemoryProviderImpl final : public MemoryProvider { + public: + std::optional> getCurrentMemory() + const override { + if (current_memory_) { + return std::reference_wrapper(*current_memory_); + } + return std::nullopt; + } + + [[nodiscard]] outcome::result resetMemory( + const MemoryConfig &) override { + current_memory_ = MemoryImpl{}; + } + + private: + std::optional> current_memory_; + }; + + class ModuleImpl : public Module, + public std::enable_shared_from_this { + public: + outcome::result> instantiate() + const override { + StoreContext store_ctx = WasmEdge_StoreCreate(); + + ModuleInstanceContext instance_ctx; + auto res = WasmEdge_ExecutorInstantiate(executor_->raw(), + &instance_ctx.raw(), + store_ctx.raw(), + module_.raw()); + if (!WasmEdge_ResultOK(res)) { + BOOST_ASSERT(WasmEdge_ResultGetCategory(res) + == WasmEdge_ErrCategory_WASM); + return std::error_code(WasmEdge_ResultGetCode(res), + WasmEdgeErrCategory{}); + } + InstanceEnvironment instance_env{ + .memory_provider = std::make_shared(), + .host_api = host_api_, + .storage_provider = storage_provider_, + .on_destruction = {}, + }; + return std::make_shared(shared_from_this(), executor_, std::move(instance_ctx), instance_env, code_hash_); + } + + private: + explicit ModuleImpl(ASTModuleContext module, + std::shared_ptr executor, + const common::Hash256 code_hash) + : module_{std::move(module)}, + executor_{executor}, + code_hash_{code_hash} { + BOOST_ASSERT(module_ != nullptr); + BOOST_ASSERT(executor_ != nullptr); + } + + ASTModuleContext module_; + std::shared_ptr executor_; + std::shared_ptr host_api_; + std::shared_ptr storage_provider_; + const common::Hash256 code_hash_; + }; + + outcome::result> ModuleFactoryImpl::make( + gsl::span code) const { + ConfigureContext configure_ctx = WasmEdge_ConfigureCreate(); + if (configure_ctx == nullptr) { + // return error + } + LoaderContext loader_ctx = WasmEdge_LoaderCreate(configure_ctx.raw()); + WasmEdge_ASTModuleContext *module_ctx; + auto res = WasmEdge_LoaderParseFromBuffer( + loader_ctx.raw(), &module_ctx, code.data(), code.size()); + ASTModuleContext module = module_ctx; + if (!WasmEdge_ResultOK(res)) { + // return error + } + } + +} // namespace kagome::runtime::wasm_edge \ No newline at end of file diff --git a/core/runtime/wasm_edge/module_factory_impl.hpp b/core/runtime/wasm_edge/module_factory_impl.hpp new file mode 100644 index 0000000000..9861d842ff --- /dev/null +++ b/core/runtime/wasm_edge/module_factory_impl.hpp @@ -0,0 +1,22 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "runtime/module_factory.hpp" + +namespace kagome::runtime::wasm_edge { + + class ModuleFactoryImpl : public ModuleFactory { + public: + + outcome::result> make( + gsl::span code) const override; + + private: + + }; + +} // namespace kagome::runtime::wasm_edge From 04e49640f08c91720fdf77dcbce6eb019daeb234 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 7 Sep 2023 17:44:39 +0300 Subject: [PATCH 21/43] Temporarily exclude WAVM --- CMakeLists.txt | 3 +- core/injector/CMakeLists.txt | 3 +- core/injector/application_injector.cpp | 142 ++++++++++-------- core/runtime/CMakeLists.txt | 2 +- .../runtime/wasm_edge/module_factory_impl.cpp | 57 +++++-- .../runtime/wasm_edge/module_factory_impl.hpp | 4 +- 6 files changed, 128 insertions(+), 83 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f9c3f31e83..21ba89b82e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,7 +97,8 @@ if("${CMAKE_CXX_COMPILER_ID}" MATCHES "^(AppleClang|Clang|GNU)$") add_flag(-Werror=unused-lambda-capture) # error if lambda capture is unused add_flag(-Werror=return-type) # warning: control reaches end of non-void function [-Wreturn-type] add_flag(-Werror=sign-compare) # warn the user if they compare a signed and unsigned numbers - add_flag(-Werror=reorder) # field '$1' will be initialized after field '$2' + # breaks soralog headers + # add_flag(-Werror=reorder) # field '$1' will be initialized after field '$2' add_flag(-Werror=mismatched-tags) # warning: class '$1' was previously declared as struct add_flag(-Werror=switch) # unhandled values in a switch statement # cmake-format: on diff --git a/core/injector/CMakeLists.txt b/core/injector/CMakeLists.txt index 5a16ae84e1..3443cce22f 100644 --- a/core/injector/CMakeLists.txt +++ b/core/injector/CMakeLists.txt @@ -8,7 +8,7 @@ add_library(application_injector ) target_link_libraries(application_injector Boost::Boost.DI - runtime_wavm + # runtime_wavm account_nonce_api address_publisher api @@ -54,6 +54,7 @@ target_link_libraries(application_injector pbkdf2_provider block_builder runtime_upgrade_tracker + runtime_wasm_edge executor runtime_properties_cache executor diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index ddca8ff279..33ac149f64 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -134,6 +134,7 @@ #include "runtime/common/storage_code_provider.hpp" #include "runtime/common/trie_storage_provider_impl.hpp" #include "runtime/executor.hpp" +#include "runtime/module.hpp" #include "runtime/module_factory.hpp" #include "runtime/runtime_api/impl/account_nonce_api.hpp" #include "runtime/runtime_api/impl/authority_discovery_api.hpp" @@ -147,18 +148,18 @@ #include "runtime/runtime_api/impl/session_keys_api.hpp" #include "runtime/runtime_api/impl/tagged_transaction_queue.hpp" #include "runtime/runtime_api/impl/transaction_payment_api.hpp" -#include "runtime/wasm_edge/module_factory_impl.hpp" #include "runtime/wasm_edge/core_api_factory_impl.hpp" -#include "runtime/wavm/compartment_wrapper.hpp" -#include "runtime/wavm/core_api_factory_impl.hpp" -#include "runtime/wavm/instance_environment_factory.hpp" -#include "runtime/wavm/intrinsics/intrinsic_functions.hpp" -#include "runtime/wavm/intrinsics/intrinsic_module.hpp" -#include "runtime/wavm/intrinsics/intrinsic_module_instance.hpp" -#include "runtime/wavm/intrinsics/intrinsic_resolver_impl.hpp" -#include "runtime/wavm/module.hpp" -#include "runtime/wavm/module_cache.hpp" -#include "runtime/wavm/module_factory_impl.hpp" +#include "runtime/wasm_edge/module_factory_impl.hpp" +// #include "runtime/wavm/compartment_wrapper.hpp" +// #include "runtime/wavm/core_api_factory_impl.hpp" +// #include "runtime/wavm/instance_environment_factory.hpp" +// #include "runtime/wavm/intrinsics/intrinsic_functions.hpp" +// #include "runtime/wavm/intrinsics/intrinsic_module.hpp" +// #include "runtime/wavm/intrinsics/intrinsic_module_instance.hpp" +// #include "runtime/wavm/intrinsics/intrinsic_resolver_impl.hpp" +// #include "runtime/wavm/module.hpp" +// #include "runtime/wavm/module_cache.hpp" +// #include "runtime/wavm/module_factory_impl.hpp" #include "storage/changes_trie/impl/storage_changes_tracker_impl.hpp" #include "storage/rocksdb/rocksdb.hpp" #include "storage/spaces.hpp" @@ -340,55 +341,63 @@ namespace { return std::make_shared("worker", cores); } - template - auto makeWavmInjector(Ts &&...args) { - return di::make_injector( - bind_by_lambda([](const auto - &injector) { - return std::make_shared( - "Runtime Compartment"); - }), - bind_by_lambda( - [](const auto &injector) { - auto compartment = injector.template create< - sptr>(); - runtime::wavm::ModuleParams module_params{}; - auto module = std::make_unique( - compartment, module_params.intrinsicMemoryType); - runtime::wavm::registerHostApiMethods(*module); - return module; - }), - bind_by_lambda( - [](const auto &injector) { - auto module = - injector - .template create>(); - return module->instantiate(); - }), - bind_by_lambda([](const auto - &injector) { - std::optional> - module_cache_opt; - auto &app_config = - injector.template create(); - if (app_config.useWavmCache()) { - module_cache_opt = std::make_shared( - injector.template create>(), - app_config.runtimeCacheDirPath()); - } - return std::make_shared( - injector - .template create>(), - injector.template create>(), - injector.template create< - sptr>(), - injector.template create>(), - module_cache_opt, - injector.template create>()); - }), - di::bind.template to(), - std::forward(args)...); - } + // template + // auto makeWavmInjector(Ts &&...args) { + // return di::make_injector( + // bind_by_lambda([](const auto + // &injector) + // { + // return + // std::make_shared( + // "Runtime Compartment"); + // }), + // bind_by_lambda( + // [](const auto &injector) { + // auto compartment = injector.template create< + // sptr>(); + // runtime::wavm::ModuleParams module_params{}; + // auto module = + // std::make_unique( + // compartment, module_params.intrinsicMemoryType); + // runtime::wavm::registerHostApiMethods(*module); + // return module; + // }), + // bind_by_lambda( + // [](const auto &injector) { + // auto module = + // injector + // .template + // create>(); + // return module->instantiate(); + // }), + // bind_by_lambda([](const auto + // &injector) { + // std::optional> + // module_cache_opt; + // auto &app_config = + // injector.template create(); + // if (app_config.useWavmCache()) { + // module_cache_opt = std::make_shared( + // injector.template create>(), + // app_config.runtimeCacheDirPath()); + // } + // return std::make_shared( + // injector + // .template + // create>(), + // injector.template create>(), + // injector.template create< + // sptr>(), + // injector.template + // create>(), + // module_cache_opt, + // injector.template create>()); + // }), + // di::bind.template + // to(), + // std::forward(args)...); + // } template auto makeBinaryenInjector(Ts &&...args) { @@ -415,7 +424,7 @@ namespace { template auto choose_runtime_implementation( @@ -428,8 +437,9 @@ namespace { return std::static_pointer_cast( injector.template create>()); case RuntimeExecutionMethod::CompileWavm: - return std::static_pointer_cast( - injector.template create>()); + return std::shared_ptr(nullptr); + // return std::static_pointer_cast( + // injector.template create>()); case RuntimeExecutionMethod::CompileWasmEdge: return std::static_pointer_cast( injector.template create>()); @@ -476,7 +486,7 @@ namespace { return injector .template create>(); }), - makeWavmInjector(), + // makeWavmInjector(), makeBinaryenInjector(), makeWasmEdgeInjector(), bind_by_lambda([](const auto &injector) { @@ -487,14 +497,14 @@ namespace { return choose_runtime_implementation< runtime::CoreApiFactory, runtime::binaryen::CoreApiFactoryImpl, - runtime::wavm::CoreApiFactoryImpl, + // runtime::wavm::CoreApiFactoryImpl, runtime::wasm_edge::CoreApiFactoryImpl>(injector, method); }), bind_by_lambda([method](const auto &injector) { return choose_runtime_implementation< runtime::ModuleFactory, runtime::binaryen::ModuleFactoryImpl, - runtime::wavm::ModuleFactoryImpl, + // runtime::wavm::ModuleFactoryImpl, runtime::wasm_edge::ModuleFactoryImpl>(injector, method); }), di::bind.template to(), diff --git a/core/runtime/CMakeLists.txt b/core/runtime/CMakeLists.txt index 5458855ab3..5366f1efe0 100644 --- a/core/runtime/CMakeLists.txt +++ b/core/runtime/CMakeLists.txt @@ -5,7 +5,7 @@ add_subdirectory(common) add_subdirectory(binaryen) -add_subdirectory(wavm) +# add_subdirectory(wavm) add_subdirectory(wasm_edge) add_subdirectory(runtime_api/impl) diff --git a/core/runtime/wasm_edge/module_factory_impl.cpp b/core/runtime/wasm_edge/module_factory_impl.cpp index 7b96961add..9ea98e1aea 100644 --- a/core/runtime/wasm_edge/module_factory_impl.cpp +++ b/core/runtime/wasm_edge/module_factory_impl.cpp @@ -24,7 +24,7 @@ OUTCOME_HPP_DECLARE_ERROR(kagome::runtime::wasm_edge, Error); OUTCOME_CPP_DEFINE_CATEGORY(kagome::runtime::wasm_edge, Error, e) { using E = kagome::runtime::wasm_edge::Error; switch (e) { - case kagome::runtime::wasm_edge::Error::INVALID_VALUE_TYPE: + case E::INVALID_VALUE_TYPE: return "Invalid value type"; } return "Unknown WasmEdge error"; @@ -244,8 +244,18 @@ namespace kagome::runtime::wasm_edge { class MemoryImpl final : public Memory { public: - MemoryImpl(MemoryInstanceContext mem_instance, MemoryAllocator allocator) - : mem_instance_{std::move(mem_instance)}, allocator_{std::move(allocator)} { + MemoryImpl(MemoryInstanceContext mem_instance, const MemoryConfig &config) + : mem_instance_{std::move(mem_instance)}, + allocator_{MemoryAllocator{ + MemoryAllocator::MemoryHandle{ + .resize = [this](size_t new_size) { resize(new_size); }, + .getSize = [this]() -> size_t { return size(); }, + .storeSz = [this](WasmPointer p, + uint32_t n) { store32(p, n); }, + .loadSz = [this](WasmPointer p) -> uint32_t { + return load32u(p); + }}, + config}} { BOOST_ASSERT(mem_instance_ != nullptr); } /** @@ -400,23 +410,36 @@ namespace kagome::runtime::wasm_edge { std::optional> getCurrentMemory() const override { if (current_memory_) { - return std::reference_wrapper(*current_memory_); + return std::reference_wrapper(**current_memory_); } return std::nullopt; } [[nodiscard]] outcome::result resetMemory( - const MemoryConfig &) override { - current_memory_ = MemoryImpl{}; + const MemoryConfig &config) override { + MemoryInstanceContext mem_instance = + WasmEdge_MemoryInstanceCreate(mem_type_); + current_memory_ = + std::make_shared(std::move(mem_instance), config); + return outcome::success(); } private: std::optional> current_memory_; + WasmEdge_MemoryTypeContext *mem_type_; }; class ModuleImpl : public Module, public std::enable_shared_from_this { public: + static std::shared_ptr create( + ASTModuleContext module, + std::shared_ptr executor, + const common::Hash256 &code_hash) { + return std::shared_ptr{ + new ModuleImpl{std::move(module), executor, code_hash}}; + } + outcome::result> instantiate() const override { StoreContext store_ctx = WasmEdge_StoreCreate(); @@ -433,18 +456,22 @@ namespace kagome::runtime::wasm_edge { WasmEdgeErrCategory{}); } InstanceEnvironment instance_env{ - .memory_provider = std::make_shared(), - .host_api = host_api_, - .storage_provider = storage_provider_, - .on_destruction = {}, + std::make_shared(), + storage_provider_, + host_api_, + {}, }; - return std::make_shared(shared_from_this(), executor_, std::move(instance_ctx), instance_env, code_hash_); + return std::make_shared(shared_from_this(), + executor_, + std::move(instance_ctx), + std::move(instance_env), + code_hash_); } private: explicit ModuleImpl(ASTModuleContext module, std::shared_ptr executor, - const common::Hash256 code_hash) + common::Hash256 code_hash) : module_{std::move(module)}, executor_{executor}, code_hash_{code_hash} { @@ -473,6 +500,12 @@ namespace kagome::runtime::wasm_edge { if (!WasmEdge_ResultOK(res)) { // return error } + + auto executor = std::make_shared( + WasmEdge_ExecutorCreate(nullptr, nullptr)); + + auto code_hash = hasher_->sha2_256(code); + return ModuleImpl::create(std::move(module), executor, code_hash); } } // namespace kagome::runtime::wasm_edge \ No newline at end of file diff --git a/core/runtime/wasm_edge/module_factory_impl.hpp b/core/runtime/wasm_edge/module_factory_impl.hpp index 9861d842ff..199bfd3014 100644 --- a/core/runtime/wasm_edge/module_factory_impl.hpp +++ b/core/runtime/wasm_edge/module_factory_impl.hpp @@ -6,17 +6,17 @@ #pragma once #include "runtime/module_factory.hpp" +#include "crypto/hasher.hpp" namespace kagome::runtime::wasm_edge { class ModuleFactoryImpl : public ModuleFactory { public: - outcome::result> make( gsl::span code) const override; private: - + std::shared_ptr hasher_; }; } // namespace kagome::runtime::wasm_edge From 96a7e18b8e8c31cdf957da202a26d9d073831209 Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 2 Oct 2023 14:44:49 +0300 Subject: [PATCH 22/43] Working version of WasmEdge --- CMakeLists.txt | 2 + cmake/Hunter/config.cmake | 13 + cmake/toolchain/compiler/gcc-13.cmake | 34 ++ cmake/toolchain/gcc-13_cxx17.cmake | 2 + .../api/service/state/impl/state_api_impl.cpp | 3 +- .../impl/app_configuration_impl.cpp | 27 +- core/common/lru_cache.hpp | 74 ++-- core/host_api/host_api.hpp | 8 +- .../host_api/impl/child_storage_extension.cpp | 2 +- .../host_api/impl/child_storage_extension.hpp | 2 +- core/host_api/impl/host_api_impl.cpp | 8 +- core/host_api/impl/host_api_impl.hpp | 8 +- core/host_api/impl/misc_extension.cpp | 9 +- core/host_api/impl/misc_extension.hpp | 2 +- core/host_api/impl/offchain_extension.cpp | 4 +- core/host_api/impl/offchain_extension.hpp | 12 +- core/injector/application_injector.cpp | 40 +- core/injector/calculate_genesis_state.hpp | 2 +- core/log/configurator.cpp | 1 - core/network/CMakeLists.txt | 1 + core/network/impl/protocols/light.cpp | 11 +- core/network/impl/protocols/light.hpp | 4 +- core/parachain/pvf/pvf_impl.cpp | 2 +- .../binaryen/core_api_factory_impl.cpp | 61 +-- .../binaryen/core_api_factory_impl.hpp | 14 +- .../binaryen/instance_environment_factory.cpp | 16 +- .../binaryen/instance_environment_factory.hpp | 15 +- core/runtime/binaryen/memory_impl.cpp | 3 + .../binaryen/module/module_factory_impl.cpp | 6 +- .../binaryen/module/module_factory_impl.hpp | 4 +- core/runtime/binaryen/module/module_impl.cpp | 11 +- core/runtime/binaryen/module/module_impl.hpp | 11 +- .../binaryen/module/module_instance_impl.cpp | 25 +- .../binaryen/module/module_instance_impl.hpp | 6 +- .../binaryen/runtime_external_interface.cpp | 10 +- core/runtime/common/executor.cpp | 21 +- core/runtime/common/executor.hpp | 0 core/runtime/common/memory_allocator.cpp | 4 +- core/runtime/common/memory_allocator.hpp | 18 +- core/runtime/common/module_instance.cpp | 7 +- .../runtime/common/module_repository_impl.cpp | 3 +- core/runtime/common/runtime_context.cpp | 16 +- .../runtime/common/runtime_instances_pool.cpp | 10 +- core/runtime/core_api_factory.hpp | 6 +- core/runtime/executor.hpp | 360 +++++++++++------- core/runtime/module_instance.hpp | 65 +++- core/runtime/runtime_api/core.hpp | 24 +- .../runtime_api/impl/account_nonce_api.cpp | 5 +- .../impl/authority_discovery_api.cpp | 14 +- core/runtime/runtime_api/impl/babe_api.cpp | 15 +- .../runtime_api/impl/block_builder.cpp | 24 +- core/runtime/runtime_api/impl/core.cpp | 47 +-- core/runtime/runtime_api/impl/core.hpp | 14 +- core/runtime/runtime_api/impl/grandpa_api.cpp | 34 +- core/runtime/runtime_api/impl/metadata.cpp | 10 +- .../runtime_api/impl/offchain_worker_api.cpp | 5 +- .../runtime_api/impl/parachain_host.cpp | 205 ++++++---- .../runtime_api/impl/session_keys_api.cpp | 10 +- .../impl/tagged_transaction_queue.cpp | 5 +- .../impl/transaction_payment_api.cpp | 9 +- core/runtime/runtime_context.hpp | 32 +- core/runtime/types.hpp | 24 +- core/runtime/wasm_edge/CMakeLists.txt | 4 +- .../wasm_edge/core_api_factory_impl.cpp | 27 +- .../wasm_edge/core_api_factory_impl.hpp | 28 +- .../runtime/wasm_edge/module_factory_impl.cpp | 305 ++++++++++----- .../runtime/wasm_edge/module_factory_impl.hpp | 40 +- core/runtime/wasm_edge/register_host_api.hpp | 287 ++++++++++++++ 68 files changed, 1437 insertions(+), 694 deletions(-) create mode 100644 cmake/toolchain/compiler/gcc-13.cmake create mode 100644 cmake/toolchain/gcc-13_cxx17.cmake delete mode 100644 core/runtime/common/executor.hpp create mode 100644 core/runtime/wasm_edge/register_host_api.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 21ba89b82e..dd60d497ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,8 @@ if("${CMAKE_CXX_COMPILER_ID}" MATCHES "^(AppleClang|Clang|GNU)$") # cmake-format: off # enable those flags + add_flag(-fdiagnostics-show-template-tree) + add_flag(-Wall) add_flag(-Wextra) add_flag(-Woverloaded-virtual) # warn if you overload (not override) a virtual function diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index c3d4035e5d..d1f74308b7 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -41,6 +41,18 @@ hunter_config( CMAKE_ARGS WITH_GFLAGS=OFF ) +hunter_config( + WasmEdge + VERSION 0.13.3 + KEEP_PACKAGE_SOURCES +) + +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(WAVM_CXX_FLAGS -Wno-redundant-move;-Wno-dangling-reference;-Wno-error=extra;) +else() + set(WAVM_CXX_FLAGS -Wno-redundant-move) +endif() + hunter_config( wavm VERSION 1.0.12 @@ -51,5 +63,6 @@ hunter_config( WAVM_BUILD_EXAMPLES=OFF WAVM_BUILD_TESTS=OFF WAVM_DISABLE_UNIX_SIGNALS=ON + WAVM_CXX_FLAGS=${WAVM_CXX_FLAGS} KEEP_PACKAGE_SOURCES ) diff --git a/cmake/toolchain/compiler/gcc-13.cmake b/cmake/toolchain/compiler/gcc-13.cmake new file mode 100644 index 0000000000..8dcbbd4eaf --- /dev/null +++ b/cmake/toolchain/compiler/gcc-13.cmake @@ -0,0 +1,34 @@ +if(DEFINED POLLY_COMPILER_GCC_13_CMAKE_) + return() +else() + set(POLLY_COMPILER_GCC_13_CMAKE_ 1) +endif() + +find_program(CMAKE_C_COMPILER gcc-13) +find_program(CMAKE_CXX_COMPILER g++-13) + +if(NOT CMAKE_C_COMPILER) + fatal_error("gcc-13 not found") +endif() + +if(NOT CMAKE_CXX_COMPILER) + fatal_error("g++-13 not found") +endif() + +set( + CMAKE_C_COMPILER + "${CMAKE_C_COMPILER}" + CACHE + STRING + "C compiler" + FORCE +) + +set( + CMAKE_CXX_COMPILER + "${CMAKE_CXX_COMPILER}" + CACHE + STRING + "C++ compiler" + FORCE +) diff --git a/cmake/toolchain/gcc-13_cxx17.cmake b/cmake/toolchain/gcc-13_cxx17.cmake new file mode 100644 index 0000000000..1d2c3f2d8b --- /dev/null +++ b/cmake/toolchain/gcc-13_cxx17.cmake @@ -0,0 +1,2 @@ +include(${CMAKE_CURRENT_LIST_DIR}/compiler/gcc-13.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/cxx17.cmake) \ No newline at end of file diff --git a/core/api/service/state/impl/state_api_impl.cpp b/core/api/service/state/impl/state_api_impl.cpp index 4f1cfa762a..9405cdb2b4 100644 --- a/core/api/service/state/impl/state_api_impl.cpp +++ b/core/api/service/state/impl/state_api_impl.cpp @@ -65,7 +65,8 @@ namespace kagome::api { const std::optional &opt_at) const { auto at = opt_at.has_value() ? opt_at.value() : block_tree_->bestLeaf().hash; - return executor_->callAt(at, method, data); + OUTCOME_TRY(ctx, executor_->ctx().ephemeralAt(at)); + return ctx.module_instance->callExportFunction(ctx, method, data); } outcome::result> StateApiImpl::getKeysPaged( diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index 2cc7c74368..296c78caba 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -5,6 +5,7 @@ #include "application/impl/app_configuration_impl.hpp" +#include #include #include #include @@ -15,7 +16,6 @@ #include #include #include -#include #include #include "api/transport/tuner.hpp" @@ -153,6 +153,19 @@ namespace { return std::nullopt; } + std::array execution_methods{ + "Interpreted", "Compiled-Wavm", "Compiled-WasmEdge"}; + + std::string execution_methods_str = []() { + std::stringstream ss; + ss << "["; + for (auto &method : execution_methods) { + ss << method << ", "; + } + ss << "]"; + return ss.str(); + }(); + std::optional str_to_runtime_exec_method(std::string_view str) { using REM = kagome::application::AppConfiguration::RuntimeExecutionMethod; @@ -838,7 +851,7 @@ namespace kagome::application { ("sync", po::value()->default_value(def_full_sync), "choose the desired sync method (Full, Fast). Full is used by default.") ("wasm-execution", po::value()->default_value(def_wasm_execution), - "choose the desired wasm execution method (Compiled, Interpreted)") + fmt::format("choose the desired wasm execution method ({})", execution_methods_str).c_str()) ("unsafe-cached-wavm-runtime", "use WAVM runtime cache") ("purge-wavm-cache", "purge WAVM runtime cache") ("parachain-runtime-instance-cache-size", @@ -1367,8 +1380,10 @@ namespace kagome::application { if (not runtime_exec_method_opt) { exec_method_value_error = true; SL_ERROR(logger_, - "Invalid runtime execution method specified: '{}'", - val); + "Invalid runtime execution method specified: '{}'. " + "Available methods are: {}", + val, + execution_methods_str); } else { runtime_exec_method_ = runtime_exec_method_opt.value(); } @@ -1395,8 +1410,8 @@ namespace kagome::application { } } - if (auto arg = - find_argument(vm, "parachain-runtime-instance-cache-size"); + if (auto arg = find_argument( + vm, "parachain-runtime-instance-cache-size"); arg.has_value()) { parachain_runtime_instance_cache_size_ = *arg; } diff --git a/core/common/lru_cache.hpp b/core/common/lru_cache.hpp index 45d3dab2f3..56fadd6ed3 100644 --- a/core/common/lru_cache.hpp +++ b/core/common/lru_cache.hpp @@ -19,49 +19,45 @@ namespace kagome { - namespace { - - template