From a451b24080dca040c4e365db197b0a450fcf9d9c Mon Sep 17 00:00:00 2001 From: kamilsa Date: Fri, 29 Nov 2024 21:35:22 +0500 Subject: [PATCH] Small improvements (#2296) * Make compiled as default option * Reduce some logs * Update hunter --------- Co-authored-by: Kirill Azovtsev --- .github/workflows/test.yml | 117 ++++++++++++------ cmake/Hunter/config.cmake | 25 +--- cmake/Hunter/hunter-gate-url.cmake | 4 +- cmake/toolchain/clang-16_cxx20.cmake | 2 - cmake/toolchain/clang-19_cxx20.cmake | 2 + ..._cxx20.cmake => clang-19_mold_cxx20.cmake} | 2 +- .../{clang-16.cmake => clang-19.cmake} | 20 +-- .../impl/app_configuration_impl.cpp | 2 +- core/crypto/twox/twox.cpp | 6 +- .../approval/approval_distribution.cpp | 61 ++++----- .../availability/fetch/fetch_impl.cpp | 2 +- .../validator/impl/parachain_processor.cpp | 2 +- .../statement_distribution.cpp | 2 +- housekeeping/docker/kagome-dev/Makefile | 30 +++-- .../kagome-dev/kagome_builder.Dockerfile | 1 + housekeeping/macos/dependency.sh | 2 +- scripts/.env | 2 +- scripts/init.sh | 2 +- 18 files changed, 158 insertions(+), 126 deletions(-) delete mode 100644 cmake/toolchain/clang-16_cxx20.cmake create mode 100644 cmake/toolchain/clang-19_cxx20.cmake rename cmake/toolchain/{clang-16_mold_cxx20.cmake => clang-19_mold_cxx20.cmake} (62%) rename cmake/toolchain/compiler/{clang-16.cmake => clang-19.cmake} (75%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 02a064ec9d..76aa425633 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,15 +69,20 @@ env: jobs: MacOS: runs-on: macos-15 - timeout-minutes: 120 + timeout-minutes: 240 strategy: fail-fast: false matrix: options: - name: "MacOS: Build Debug" build_type: "Debug" + wasm_compiler: "WasmEdge" - name: "MacOS: Build Release" build_type: "Release" + wasm_compiler: "WasmEdge" + - name: "MacOS WAVM: Build Debug" + build_type: "Debug" + wasm_compiler: "WAVM" name: "${{ matrix.options.name }}" steps: - uses: actions/checkout@v4 @@ -91,12 +96,11 @@ jobs: env: KAGOME_MAC_CI: 1 - name: build - run: ./housekeeping/make_build.sh -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.options.build_type }} -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake + run: ./housekeeping/make_build.sh -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.options.build_type }} -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=${{ matrix.options.wasm_compiler }} -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake env: KAGOME_IN_DOCKER: 0 KAGOME_MAC_CI: 1 - Linux: if: false # Need to fix # ${{ @@ -106,12 +110,12 @@ jobs: fail-fast: false matrix: options: - - name: "Linux: clang-16 External Project" - run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake + - name: "Linux: clang-19 External Project" + run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-19_cxx20.cmake name: "${{ matrix.options.name }}" runs-on: ubuntu-24.04 timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb + container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19 steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 @@ -133,44 +137,12 @@ jobs: df -m || true du -hd1 /__w /github || true - Linux-self-hosted: - strategy: - fail-fast: false - matrix: - options: - - name: "Self-hosted: Linux: gcc-13 ASAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON - - name: "Self-hosted: Linux: clang-16 TSAN WAVM" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM - - name: "Self-hosted: Linux: clang-16 UBSAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON -DUBSAN_TRAP=OFF -DUBSAN_ABORT=ON - env: UBSAN_OPTIONS=print_stacktrace=1 - # Need to fix - # - name: "Self-hosted: Linux: clang-16 External Project" - # run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake - - name: "${{ matrix.options.name }}" - runs-on: [ actions-runner-controller ] - timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb - steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - if: ${{ env.USE_CACHE == 'true' }} - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - - name: "${{ matrix.options.name }}" - run: "${{ matrix.options.run }}" - coverage-self-hosted: if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} name: "Self-hosted: Linux: gcc-13 coverage/sonar" runs-on: [ actions-runner-controller ] timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb + container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19 steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 @@ -202,6 +174,73 @@ jobs: BRANCH_NAME: ${{ github.ref }} run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi + kagome_dev_docker_build_sanitizers: + strategy: + fail-fast: false + matrix: + options: + - name: "Self-hosted: Linux: gcc-13 ASAN" + params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON + - name: "Self-hosted: Linux: clang-19 TSAN WAVM" + params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-19_cxx20.cmake -DTSAN=ON + - name: "Self-hosted: Linux: clang-19 UBSAN" + params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-19_cxx20.cmake -DUBSAN=ON -DUBSAN_TRAP=OFF -DUBSAN_ABORT=ON + + name: "${{ matrix.options.name }}" + runs-on: [ actions-runner-controller ] + timeout-minutes: 180 + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Get master branch" + if: github.ref != 'refs/heads/master' + run: git fetch origin master:master || true + + - name: "Get commit version" + working-directory: ./housekeeping/docker/kagome-dev + run: make get_versions + + - name: "Check version" + working-directory: ./housekeeping/docker/kagome-dev + run: | + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + echo "short_commit_hash=${SHORT_COMMIT_HASH}" | tee $GITHUB_ENV + + - name: "Cache dependencies" + id: cache-restore + if: ${{ env.USE_CACHE == 'true' }} + uses: actions/cache/restore@v4 + with: + path: ${{ env.CACHE_PATH }} + key: ${{ github.job }}-${{ env.CACHE_VERSION }}-sanitizers-${{ env.short_commit_hash }} + restore-keys: | + ${{ github.job }}-${{ env.CACHE_VERSION }}-sanitizers- + + - name: "Build target" + working-directory: ./housekeeping/docker/kagome-dev + run: + make kagome_dev_docker_build_sanitizers \ + GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \ + GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \ + BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \ + CI="true" + SAN_PARAMS="${{ matrix.options.params }}" + + - name: "Cleaning cache" + run: | + find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \; + + - name: "Always Save Cache" + id: cache-save + if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True') + uses: actions/cache/save@v4 + with: + path: ${{ env.CACHE_PATH }} + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + kagome_dev_docker_build_tidy: runs-on: ubuntu-24.04 timeout-minutes: 600 # TODO(xDimon): revert after merge PR#2208 diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index 8c4c514445..bbd1efe960 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -25,13 +25,6 @@ hunter_config( KEEP_PACKAGE_SOURCES ) -hunter_config( - benchmark - URL https://github.com/google/benchmark/archive/refs/tags/v1.8.3.zip - SHA1 bf9870756ee3f8d2d3b346b24ee3600a41c74d3d - CMAKE_ARGS BENCHMARK_ENABLE_TESTING=OFF -) - hunter_config( rocksdb VERSION 9.6.1 @@ -99,8 +92,8 @@ endif () hunter_config( kagome-crates - URL https://github.com/qdrvm/kagome-crates/archive/refs/tags/1.0.2.tar.gz - SHA1 946c48508545380e155ab831be54228b916544d3 + URL https://github.com/qdrvm/kagome-crates/archive/refs/tags/v1.0.3.tar.gz + SHA1 4207446a0e45764b814805821aa6860924b03cb7 ) hunter_config( @@ -112,29 +105,23 @@ hunter_config( hunter_config( libp2p - URL https://github.com/libp2p/cpp-libp2p/archive/c96d45f792fafd6970a7e37ec816b02a9167e2b6.zip - SHA1 884932112bc75996eeecd4a7bbcb932565fe9859 + VERSION 0.1.28 ) hunter_config( scale - URL https://github.com/qdrvm/scale-codec-cpp/archive/e1a3c7afafc2eeda0c8e2daed08da6b7789f44b3.zip - SHA1 b56bcda34fb0d293c88d8b642b2f3fdc2d16a3e5 + VERSION 1.1.4 ) hunter_config( erasure_coding_crust -# VERSION 0.0.8 - URL https://github.com/qdrvm/erasure-coding-crust/archive/refs/tags/v0.0.8.tar.gz - SHA1 6bcdb6327f5da2dcec5c70f2fa63b95a44925af0 + VERSION 0.0.8 KEEP_PACKAGE_SOURCES ) hunter_config( soralog -# VERSION 0.2.4 - URL https://github.com/qdrvm/soralog/archive/refs/tags/v0.2.4.tar.gz - SHA1 1de495d8a3a73c1e940be3fdddf263a2d673aec1 + VERSION 0.2.4 KEEP_PACKAGE_SOURCES ) diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake index 1be9836f94..83591dcef4 100644 --- a/cmake/Hunter/hunter-gate-url.cmake +++ b/cmake/Hunter/hunter-gate-url.cmake @@ -1,5 +1,5 @@ HunterGate( - URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm23.zip - SHA1 3fb58496e599fa3d35e94d1810324c6b379029f1 + URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm25.zip + SHA1 59c66ff04ebd2cbdf86c3b996d38d4be6eaaa78b LOCAL ) \ No newline at end of file diff --git a/cmake/toolchain/clang-16_cxx20.cmake b/cmake/toolchain/clang-16_cxx20.cmake deleted file mode 100644 index 16081adfc7..0000000000 --- a/cmake/toolchain/clang-16_cxx20.cmake +++ /dev/null @@ -1,2 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-16.cmake) -include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake) diff --git a/cmake/toolchain/clang-19_cxx20.cmake b/cmake/toolchain/clang-19_cxx20.cmake new file mode 100644 index 0000000000..ff37710c7c --- /dev/null +++ b/cmake/toolchain/clang-19_cxx20.cmake @@ -0,0 +1,2 @@ +include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-19.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake) diff --git a/cmake/toolchain/clang-16_mold_cxx20.cmake b/cmake/toolchain/clang-19_mold_cxx20.cmake similarity index 62% rename from cmake/toolchain/clang-16_mold_cxx20.cmake rename to cmake/toolchain/clang-19_mold_cxx20.cmake index 86c85ef3b4..2bfc4cae5f 100644 --- a/cmake/toolchain/clang-16_mold_cxx20.cmake +++ b/cmake/toolchain/clang-19_mold_cxx20.cmake @@ -1,3 +1,3 @@ -include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-16.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-19.cmake) include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake) include(${CMAKE_CURRENT_LIST_DIR}/linker/mold.cmake) diff --git a/cmake/toolchain/compiler/clang-16.cmake b/cmake/toolchain/compiler/clang-19.cmake similarity index 75% rename from cmake/toolchain/compiler/clang-16.cmake rename to cmake/toolchain/compiler/clang-19.cmake index 61c25e46f2..ce8062443f 100644 --- a/cmake/toolchain/compiler/clang-16.cmake +++ b/cmake/toolchain/compiler/clang-19.cmake @@ -1,7 +1,7 @@ -if(DEFINED POLLY_COMPILER_CLANG_16_CMAKE) +if(DEFINED POLLY_COMPILER_CLANG_19_CMAKE) return() else() - set(POLLY_COMPILER_CLANG_16_CMAKE 1) + set(POLLY_COMPILER_CLANG_19_CMAKE 1) endif() include(${CMAKE_CURRENT_LIST_DIR}/../../print.cmake) @@ -13,29 +13,29 @@ if(XCODE_VERSION) fatal_error(${_err}) endif() -find_program(CMAKE_C_COMPILER clang-16) -find_program(CMAKE_CXX_COMPILER clang++-16) +find_program(CMAKE_C_COMPILER clang-19) +find_program(CMAKE_CXX_COMPILER clang++-19) if (CMAKE_CXX_COMPILER STREQUAL "CMAKE_CXX_COMPILER-NOTFOUND") - message(STATUS "clang++-16 not found, checking clang++") + message(STATUS "clang++-19 not found, checking clang++") cmake_path(GET CMAKE_C_COMPILER PARENT_PATH compiler_path) message(STATUS "Assumed compiler path: ${compiler_path}") - # clang++-16 doesn't always exist + # clang++-19 doesn't always exist find_program(CMAKE_CXX_COMPILER clang++ PATHS "${compiler_path}" NO_DEFAULT_PATH REQUIRED) execute_process(COMMAND "${CMAKE_CXX_COMPILER}" --version OUTPUT_VARIABLE compiler_version_output) string(REGEX MATCH "clang version ([0-9]+)\\.[0-9]+\\.[0-9]+" compiler_version "${compiler_version_output}") - if (NOT CMAKE_MATCH_1 STREQUAL "16") - message(FATAL_ERROR "Found clang++ version ${CMAKE_MATCH_1}, 16 is required") + if (NOT CMAKE_MATCH_1 STREQUAL "19") + message(FATAL_ERROR "Found clang++ version ${CMAKE_MATCH_1}, 19 is required") endif() endif() if(NOT CMAKE_C_COMPILER) - fatal_error("clang-16 not found") + fatal_error("clang-19 not found") endif() if(NOT CMAKE_CXX_COMPILER) - fatal_error("clang++-16 not found") + fatal_error("clang++-19 not found") endif() set( diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index 9ca33b02e3..c19b473cc0 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -90,7 +90,7 @@ namespace { }(); const auto def_sync_method = kagome::application::SyncMethod::Full; const auto def_runtime_exec_method = - kagome::application::AppConfiguration::RuntimeExecutionMethod::Interpret; + kagome::application::AppConfiguration::RuntimeExecutionMethod::Compile; const auto def_runtime_interpreter = kagome::application::AppConfiguration::RuntimeInterpreter::WasmEdge; const auto def_purge_wavm_cache_ = false; diff --git a/core/crypto/twox/twox.cpp b/core/crypto/twox/twox.cpp index ac47c25e55..13903e2a7b 100644 --- a/core/crypto/twox/twox.cpp +++ b/core/crypto/twox/twox.cpp @@ -39,8 +39,11 @@ namespace kagome::crypto { } void make_twox256(const uint8_t *in, uint32_t len, uint8_t *out) { + // Ensure the buffer is aligned to the boundary required for uint64_t + // (required for happy UBSAN) + std::array aligned_out{}; // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - auto *ptr = reinterpret_cast(out); + auto *ptr = reinterpret_cast(aligned_out.data()); // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) ptr[0] = XXH64(in, len, 0); // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) @@ -49,6 +52,7 @@ namespace kagome::crypto { ptr[2] = XXH64(in, len, 2); // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) ptr[3] = XXH64(in, len, 3); + std::memcpy(out, aligned_out.data(), 4 * sizeof(uint64_t)); } common::Hash256 make_twox256(common::BufferView buf) { diff --git a/core/parachain/approval/approval_distribution.cpp b/core/parachain/approval/approval_distribution.cpp index 03f701184b..13e3622bb1 100644 --- a/core/parachain/approval/approval_distribution.cpp +++ b/core/parachain/approval/approval_distribution.cpp @@ -588,7 +588,7 @@ namespace kagome::parachain { common::MainThreadPool &main_thread_pool, LazySPtr dispute_coordinator) : approval_thread_handler_{poolHandlerReadyMake( - this, app_state_manager, approval_thread_pool, logger_)}, + this, app_state_manager, approval_thread_pool, logger_)}, worker_pool_handler_{worker_thread_pool.handler(*app_state_manager)}, parachain_host_(std::move(parachain_host)), slots_util_(slots_util), @@ -1657,17 +1657,18 @@ namespace kagome::parachain { const auto &candidate_receipt = hashed_candidate.get(); if (!opt_result) { // Unavailable - self->logger_->warn( - "No available parachain data.(session index={}, candidate " - "hash={}, relay block hash={})", - session_index, - hashed_candidate.getHash(), - relay_block_hash); + SL_DEBUG(self->logger_, + "No available parachain data. (session index={}, " + "candidate hash={}, relay block hash={})", + session_index, + hashed_candidate.getHash(), + relay_block_hash); return; } if (opt_result->has_error()) { - self->logger_->warn( + SL_WARN( + self->logger_, "Parachain data recovery failed.(error={}, session index={}, " "candidate hash={}, relay block hash={})", opt_result->error(), @@ -1685,7 +1686,8 @@ namespace kagome::parachain { auto result = self->parachain_host_->validation_code_by_hash( block_hash, candidate_receipt.descriptor.validation_code_hash); if (result.has_error() || !result.value()) { - self->logger_->warn( + SL_WARN( + self->logger_, "Approval state is failed. Block hash {}, session index {}, " "validator index {}, relay parent {}", block_hash, @@ -1695,12 +1697,12 @@ namespace kagome::parachain { return; /// ApprovalState::failed } - self->logger_->info( - "Make exhaustive validation. Candidate hash {}, validator index " - "{}, block hash {}", - hashed_candidate.getHash(), - validator_index, - block_hash); + SL_DEBUG(self->logger_, + "Make exhaustive validation. Candidate hash {}, validator " + "index {}, block hash {}", + hashed_candidate.getHash(), + validator_index, + block_hash); runtime::ValidationCode &validation_code = *result.value(); @@ -1726,12 +1728,12 @@ namespace kagome::parachain { } }); if (outcome.has_error()) { - self->logger_->warn( - "Approval validation failed.(parachain id={}, relay " - "parent={}, error={})", - candidate_receipt.descriptor.para_id, - candidate_receipt.descriptor.relay_parent, - outcome.error()); + SL_WARN(self->logger_, + "Approval validation failed.(parachain id={}, relay " + "parent={}, error={})", + candidate_receipt.descriptor.para_id, + candidate_receipt.descriptor.relay_parent, + outcome.error()); self->dispute_coordinator_.get()->issueLocalStatement( session_index, hashed_candidate.getHash(), @@ -2670,10 +2672,10 @@ namespace kagome::parachain { REINVOKE( *main_pool_handler_, runDistributeApproval, vote, std::move(peers)); - SL_INFO(logger_, - "Sending an approval to peers. (block={}, num peers={})", - vote.payload.payload.block_hash, - peers.size()); + SL_DEBUG(logger_, + "Sending an approval to peers. (block={}, num peers={})", + vote.payload.payload.block_hash, + peers.size()); router_->getValidationProtocol()->write(peers, network::vstaging::Approvals{ @@ -2892,9 +2894,10 @@ namespace kagome::parachain { }; return approval::min_or_some( e.next_no_show, - (e.last_assignment_tick ? filter( - *e.last_assignment_tick + kApprovalDelay, tick_now) - : std::optional{})); + (e.last_assignment_tick + ? filter(*e.last_assignment_tick + kApprovalDelay, + tick_now) + : std::optional{})); }, [&](const approval::PendingRequiredTranche &e) { std::optional next_announced{}; @@ -3160,7 +3163,7 @@ namespace kagome::parachain { auto opt_candidate_entry = storedCandidateEntries().get(candidate_hash); if (!opt_block_entry || !opt_candidate_entry) { - SL_ERROR(logger_, "Block entry or candidate entry not exists."); + SL_TRACE(logger_, "Block entry or candidate entry not exists."); return; } diff --git a/core/parachain/availability/fetch/fetch_impl.cpp b/core/parachain/availability/fetch/fetch_impl.cpp index f70297233c..2ffdeb6246 100644 --- a/core/parachain/availability/fetch/fetch_impl.cpp +++ b/core/parachain/availability/fetch/fetch_impl.cpp @@ -154,7 +154,7 @@ namespace kagome::parachain { } return; } - SL_WARN(log(), + SL_DEBUG(log(), "candidate={} chunk={} not found", candidate_hash, active.chunk_index); diff --git a/core/parachain/validator/impl/parachain_processor.cpp b/core/parachain/validator/impl/parachain_processor.cpp index eb819d6e6f..9bc9a49d54 100644 --- a/core/parachain/validator/impl/parachain_processor.cpp +++ b/core/parachain/validator/impl/parachain_processor.cpp @@ -2027,7 +2027,7 @@ namespace kagome::parachain { rp_state.table_context, rp_state.inject_core_index)) { const auto para_id = backed->candidate.descriptor.para_id; - SL_INFO( + SL_DEBUG( logger_, "Candidate backed.(candidate={}, para id={}, relay_parent={})", summary->candidate, diff --git a/core/parachain/validator/statement_distribution/statement_distribution.cpp b/core/parachain/validator/statement_distribution/statement_distribution.cpp index 52d25e589c..c787e8842c 100644 --- a/core/parachain/validator/statement_distribution/statement_distribution.cpp +++ b/core/parachain/validator/statement_distribution/statement_distribution.cpp @@ -1070,7 +1070,7 @@ namespace kagome::parachain::statement_distribution { [[maybe_unused]] const auto disabled_mask = parachain_state->get().disabled_bitmask(*group); const network::vstaging::AttestedCandidateResponse &response = r.value(); - SL_INFO(logger, + SL_DEBUG(logger, "Fetch attested candidate success. (relay parent={}, " "candidate={}, group index={}, statements={})", relay_parent, diff --git a/housekeeping/docker/kagome-dev/Makefile b/housekeeping/docker/kagome-dev/Makefile index 0c0d985876..6d7365bfe1 100644 --- a/housekeeping/docker/kagome-dev/Makefile +++ b/housekeeping/docker/kagome-dev/Makefile @@ -28,6 +28,8 @@ GITHUB_HUNTER_USERNAME ?= GITHUB_HUNTER_TOKEN ?= CTEST_OUTPUT_ON_FAILURE ?= 1 WERROR ?= OFF +SAN_PARAMS ?= -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON +UBSAN_OPTIONS ?= print_stacktrace=1 # Generated versions OS_IMAGE ?= $(OS_IMAGE_NAME):$(OS_IMAGE_TAG)@sha256:$(OS_IMAGE_HASH) @@ -52,6 +54,7 @@ REGION ?= europe-north1 IS_MAIN_OR_TAG ?= false GIT_REF_NAME ?= CI ?= false +BUILD_THREADS=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu) export DOCKER_BUILDKIT=1 # BUILDKIT_PROGRESS - auto, plain, tty, rawjson @@ -76,7 +79,8 @@ kagome_builder: --build-arg LLVM_VERSION=$(LLVM_VERSION) \ --build-arg BASE_IMAGE=$(OS_IMAGE_NAME) \ --build-arg BASE_IMAGE_TAG=$(OS_IMAGE_TAG_WITH_HASH) \ - --build-arg ARCHITECTURE=$(ARCHITECTURE) . + --build-arg ARCHITECTURE=$(ARCHITECTURE) \ + --no-cache . kagome_builder_push: docker push $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(BUILDER_IMAGE_TAG) ; \ @@ -97,7 +101,6 @@ kagome_dev_docker_build: $(CACHE_DIR)/.cache/ccache ; \ CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ - BUILD_THREADS=$$(nproc 2>/dev/null || sysctl -n hw.ncpu); \ DOCKER_EXEC_RESULT=0 ; \ echo "Build type: $(BUILD_TYPE)"; \ docker run -d --name $$CONTAINER_NAME \ @@ -105,7 +108,6 @@ kagome_dev_docker_build: --entrypoint "/bin/bash" \ -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ -e BUILD_TYPE=$(BUILD_TYPE) \ - -e BUILD_THREADS=$$BUILD_THREADS \ -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ @@ -126,7 +128,7 @@ kagome_dev_docker_build: git submodule update --init && \ echo \"Building in $$(pwd)\" && \ cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ - cmake --build \"$(BUILD_DIR)\" --target kagome -- -j${BUILD_THREADS} && \ + cmake --build \"$(BUILD_DIR)\" --target kagome -- -j$(BUILD_THREADS) && \ mkdir -p /tmp/kagome && \ cp /opt/kagome/$(BUILD_DIR)/node/kagome /tmp/kagome/kagome && \ cd /opt/kagome/housekeeping/docker/kagome-dev && \ @@ -154,7 +156,7 @@ kagome_dev_docker_build: fi; \ docker stop $$CONTAINER_NAME -kagome_dev_docker_build_tidy_inhouse: +kagome_dev_docker_build_sanitizers: $(MAKE) get_versions mkdir -p \ $(CACHE_DIR)/.cargo/git \ @@ -163,7 +165,6 @@ kagome_dev_docker_build_tidy_inhouse: $(CACHE_DIR)/.cache/ccache ; \ CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ - BUILD_THREADS=$$(nproc 2>/dev/null || sysctl -n hw.ncpu); \ DOCKER_EXEC_RESULT=0 ; \ echo "Build type: $(BUILD_TYPE)"; \ docker run -d --name $$CONTAINER_NAME \ @@ -171,18 +172,17 @@ kagome_dev_docker_build_tidy_inhouse: --entrypoint "/bin/bash" \ -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ -e BUILD_TYPE=$(BUILD_TYPE) \ - -e BUILD_THREADS=$$BUILD_THREADS \ -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ -e CTEST_OUTPUT_ON_FAILURE=$(CTEST_OUTPUT_ON_FAILURE) \ + -e UBSAN_OPTIONS=$(UBSAN_OPTIONS) \ -v $$(pwd)/../../../../kagome:/opt/kagome \ - -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ -v $(CACHE_DIR)/.cargo/git:/root/.cargo/git \ -v $(CACHE_DIR)/.cargo/registry:/root/.cargo/registry \ -v $(CACHE_DIR)/.hunter:/root/.hunter \ -v $(CACHE_DIR)/.cache/ccache:/root/.cache/ccache \ - $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(BUILDER_IMAGE_TAG) \ + $(DOCKERHUB_BUILDER_PATH):$(BUILDER_IMAGE_TAG) \ -c "tail -f /dev/null"; \ docker exec -t $$CONTAINER_NAME /bin/bash -c \ "clang --version && \ @@ -191,10 +191,10 @@ kagome_dev_docker_build_tidy_inhouse: git config --global --add safe.directory /root/.hunter/_Base/Cache/meta && \ source /venv/bin/activate && \ git submodule update --init && \ - echo \"Building in $$(pwd)\" && \ - cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ - cmake --build \"$(BUILD_DIR)\" --target generated -- -j${BUILD_THREADS} && \ - cd /opt/kagome/ && export CI='$(CI)' && ./housekeeping/clang-tidy-diff.sh \ + echo \"Building in \`pwd\`\" && \ + cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DBACKWARD=OFF $(SAN_PARAMS) -DWERROR=$(WERROR) && \ + cmake --build build -- -j$(BUILD_THREADS) && \ + cmake --build \"$(BUILD_DIR)\" --target test \ " || DOCKER_EXEC_RESULT=$$? ; \ if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ @@ -212,7 +212,6 @@ kagome_dev_docker_build_tidy: $(CACHE_DIR)/.cache/ccache ; \ CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ - BUILD_THREADS=$$(nproc 2>/dev/null || sysctl -n hw.ncpu); \ DOCKER_EXEC_RESULT=0 ; \ echo "Build type: $(BUILD_TYPE)"; \ docker run -d --name $$CONTAINER_NAME \ @@ -220,7 +219,6 @@ kagome_dev_docker_build_tidy: --entrypoint "/bin/bash" \ -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ -e BUILD_TYPE=$(BUILD_TYPE) \ - -e BUILD_THREADS=$$BUILD_THREADS \ -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ @@ -241,7 +239,7 @@ kagome_dev_docker_build_tidy: git submodule update --init && \ echo \"Building in $$(pwd)\" && \ cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ - cmake --build \"$(BUILD_DIR)\" --target generated -- -j${BUILD_THREADS} && \ + cmake --build \"$(BUILD_DIR)\" --target generated -- -j$(BUILD_THREADS) && \ cd /opt/kagome/ && export CI='$(CI)' && ./housekeeping/clang-tidy-diff.sh \ " || DOCKER_EXEC_RESULT=$$? ; \ if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ diff --git a/housekeeping/docker/kagome-dev/kagome_builder.Dockerfile b/housekeeping/docker/kagome-dev/kagome_builder.Dockerfile index c91b1dcc68..a248f2384a 100644 --- a/housekeeping/docker/kagome-dev/kagome_builder.Dockerfile +++ b/housekeeping/docker/kagome-dev/kagome_builder.Dockerfile @@ -41,6 +41,7 @@ RUN echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://ap RUN install_packages \ build-essential \ + ninja-build \ ccache \ clang-format-${LLVM_VERSION} \ clang-tidy-${LLVM_VERSION} \ diff --git a/housekeeping/macos/dependency.sh b/housekeeping/macos/dependency.sh index 55148577eb..fd8532d0b5 100755 --- a/housekeeping/macos/dependency.sh +++ b/housekeeping/macos/dependency.sh @@ -15,5 +15,5 @@ sudo python3 -m pip install --upgrade pip sudo python3 -m pip install scikit-build sudo python3 -m pip install cmake==3.25 requests gitpython gcovr -curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.75.0 --profile minimal +curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.81.0 --profile minimal brew install ninja diff --git a/scripts/.env b/scripts/.env index 5dd6309bad..dbc079519e 100644 --- a/scripts/.env +++ b/scripts/.env @@ -1,4 +1,4 @@ -RUST_VERSION=1.77.0 +RUST_VERSION=1.81.0 RUSTUP_HOME=~/.rustup CARGO_HOME=~/.cargo diff --git a/scripts/init.sh b/scripts/init.sh index a4a79d349c..11e3236acd 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -9,7 +9,7 @@ set -a; source $current_dir/.env; set +a #include .env vars apt update apt install --no-install-recommends -y \ build-essential git gcc ca-certificates python-is-python3 python3-pip \ - python3-venv curl libgmp-dev libncurses6 libnsl-dev libseccomp-dev + python3-venv curl libgmp-dev libncurses6 libncurses-dev libnsl-dev libseccomp-dev python3 -m venv "$parent_dir/venv"