Update libs, gcc-13, macos 15 (#2233)

* Update to macos 15 * Bump gcc to 13 * Update libp2p to master --------- Co-authored-by: Harrm <[email protected]> Co-authored-by: Kirill Azovtsev <[email protected]>
qdrvm · Oct 15, 2024 · 474f31d · 474f31d
1 parent 32c5b68
commit 474f31d
Show file tree

Hide file tree

Showing 12 changed files with 71 additions and 55 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -56,8 +56,17 @@ env:
 
 jobs:
   MacOS:
-    runs-on: macos-14
+    runs-on: macos-15
     timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        options:
+          - name: "MacOS: Build Debug"
+            build_type: "Debug"
+          - name: "MacOS: Build Release"
+            build_type: "Release"
+    name: "${{ matrix.options.name }}"
     steps:
     - uses: actions/checkout@v4
     - uses: actions/cache@v4
@@ -70,7 +79,7 @@ jobs:
       env:
         KAGOME_MAC_CI: 1
     - name: build
-      run: ./housekeeping/make_build.sh -G Ninja -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake
+      run: ./housekeeping/make_build.sh -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.options.build_type }} -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake
       env:
         KAGOME_IN_DOCKER: 0
         KAGOME_MAC_CI: 1
@@ -88,7 +97,7 @@ jobs:
           - name: "Linux: clang-16 External Project"
             run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake
     name: "${{ matrix.options.name }}"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 120
     container: qdrvm/kagome-dev:9-minideb
     steps:
@@ -117,8 +126,8 @@ jobs:
       fail-fast: false
       matrix:
         options:
-          - name: "Self-hosted: Linux: gcc-12 ASAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON
+          - name: "Self-hosted: Linux: gcc-13 ASAN"
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON
           - name: "Self-hosted: Linux: clang-16 TSAN WAVM"
             run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM
           - name: "Self-hosted: Linux: clang-16 UBSAN"
@@ -145,7 +154,7 @@ jobs:
 
   coverage-self-hosted:
     if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted')  }}
-    name: "Self-hosted: Linux: gcc-12 coverage/sonar"
+    name: "Self-hosted: Linux: gcc-13 coverage/sonar"
     runs-on: [ actions-runner-controller ]
     timeout-minutes: 120
     container: qdrvm/kagome-dev:9-minideb
@@ -161,7 +170,7 @@ jobs:
       - name: makeBuild
         env:
           BUILD_FINAL_TARGET: ctest_coverage
-        run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON
+        run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON
       - if: ${{ 
           github.event_name != 'pull_request' || 
           github.event.pull_request.head.repo.full_name == github.repository }}
@@ -180,36 +189,8 @@ jobs:
           BRANCH_NAME: ${{ github.ref }}
         run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi
 
-  Minideb:
-    if: ${{ 
-      !(github.ref == 'refs/heads/master' || 
-      startsWith( github.ref, 'refs/tags/' )) }}
-    strategy:
-      fail-fast: false
-      matrix:
-        options:
-          - name: "Minideb: Build Debug"
-            build-type: "Debug"
-          - name: "Minideb: Build Release"
-            build-type: "Release"
-    name: "${{ matrix.options.name }}"
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    container: qdrvm/kagome-dev:9-minideb
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/cache@v4
-        if: ${{ env.USE_CACHE == 'true' }}
-        with:
-          path: ${{ env.CACHE_PATHS }}
-          key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }}
-      - name: "${{ matrix.options.name }}"
-        env:
-          BUILD_TYPE: "${{ matrix.options.build-type }}"
-        run: ./housekeeping/docker/kagome-dev/make.sh
-
   kagome_dev_docker_build_tidy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 600 # TODO(xDimon): revert after merge PR#2208
     name: "Linux: clang-tidy"
 

diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake
@@ -34,7 +34,7 @@ hunter_config(
 
 hunter_config(
     rocksdb
-    VERSION 9.0.0
+    VERSION 9.6.1
     CMAKE_ARGS WITH_GFLAGS=OFF
 )
 
@@ -107,8 +107,8 @@ hunter_config(
 
 hunter_config(
     libp2p
-    URL https://github.com/libp2p/cpp-libp2p/archive/66764acb294517f8249aea6d63c6e6cc0be5686f.tar.gz
-    SHA1 45b73da05e1b59f46b9f4cb39a24c485ee6d5ba1
+    URL https://github.com/libp2p/cpp-libp2p/archive/a820e24a88170bdcd7e93c821b6b3bd6f5f01a9b.tar.gz
+    SHA1 3bce83beb222906b390e73956a47fb4820729bce
 )
 
 hunter_config(

diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake
@@ -1,5 +1,5 @@
 HunterGate(
-  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm22.zip
-  SHA1 72b046fe42baefb968898e46f9442a7e9bc4e1b0
+  URL  https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm23.zip
+  SHA1 3fb58496e599fa3d35e94d1810324c6b379029f1
   LOCAL
 )
diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake
@@ -14,7 +14,7 @@ endif ()
 
 # https://docs.hunter.sh/en/latest/packages/pkg/Boost.html
 hunter_add_package(Boost COMPONENTS random filesystem program_options date_time)
-find_package(Boost CONFIG REQUIRED random filesystem program_options date_time)
+find_package(Boost CONFIG REQUIRED random filesystem program_options date_time process)
 
 hunter_add_package(qtils)
 find_package(qtils CONFIG REQUIRED)

diff --git a/core/authority_discovery/query/query_impl.cpp b/core/authority_discovery/query/query_impl.cpp
@@ -195,8 +195,10 @@ namespace kagome::authority_discovery {
       queue_.pop_back();
 
       scheduler_->schedule([wp{weak_from_this()},
-                            hash = common::Buffer{crypto::sha256(authority)}] {
+                            hash = common::Buffer{crypto::sha256(authority)},
+                            authority] {
         if (auto self = wp.lock()) {
+          SL_DEBUG(self->log_, "start lookup({})", common::hex_lower(authority));
           std::ignore = self->kademlia_.get()->getValue(
               hash, [=](const outcome::result<std::vector<uint8_t>> &res) {
                 if (auto self = wp.lock()) {
@@ -213,6 +215,10 @@ namespace kagome::authority_discovery {
   outcome::result<void> QueryImpl::add(
       const primitives::AuthorityDiscoveryId &authority,
       outcome::result<std::vector<uint8_t>> _res) {
+    SL_TRACE(log_,
+            "lookup : add addresses for authority {}, _res {}",
+            common::hex_lower(authority),
+            _res.has_value() ? "ok" : "error: " + _res.error().message());
     OUTCOME_TRY(signed_record_pb, _res);
     auto it = auth_to_peer_cache_.find(authority);
     if (it != auth_to_peer_cache_.end()
@@ -224,6 +230,9 @@ namespace kagome::authority_discovery {
             signed_record_pb.data(),
             // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions)
             signed_record_pb.size())) {
+      SL_ERROR(log_,
+               "lookup: can't parse signed record from authority {}",
+               authority);
       return Error::DECODE_ERROR;
     }
 
@@ -241,9 +250,11 @@ namespace kagome::authority_discovery {
 
     ::authority_discovery_v3::AuthorityRecord record;
     if (not record.ParseFromString(signed_record.record())) {
+      SL_TRACE(log_, "lookup: can't parse record from authority {}", authority);
       return Error::DECODE_ERROR;
     }
     if (record.addresses().empty()) {
+      SL_ERROR(log_, "lookup: no addresses from authority {}", authority);
       return Error::NO_ADDRESSES;
     }
     std::optional<Timestamp> time{};
@@ -253,18 +264,27 @@ namespace kagome::authority_discovery {
                       qtils::str2byte(record.creation_time().timestamp())));
       time = *tmp;
       if (it != auth_to_peer_cache_.end() and time <= it->second.time) {
+        SL_TRACE(log_, "lookup: outdated record for authority {}", authority);
         return outcome::success();
       }
     }
     libp2p::peer::PeerInfo peer{.id = std::move(peer_id)};
     auto peer_id_str = peer.id.toBase58();
+    SL_TRACE(log_,
+            "lookup: adding {} addresses for authority {}",
+            record.addresses().size(),
+            authority);
     for (auto &pb : record.addresses()) {
       OUTCOME_TRY(address, libp2p::multi::Multiaddress::create(str2byte(pb)));
       auto id = address.getPeerId();
       if (not id) {
         continue;
       }
       if (id != peer_id_str) {
+        SL_ERROR(log_,
+                 "lookup: inconsistent peer id {} != {}",
+                 id.value(),
+                 peer_id_str);
         return Error::INCONSISTENT_PEER_ID;
       }
       peer.addresses.emplace_back(std::move(address));
@@ -274,6 +294,7 @@ namespace kagome::authority_discovery {
                 sr_crypto_provider_->verify(
                     auth_sig, str2byte(signed_record.record()), authority));
     if (not auth_sig_ok) {
+      SL_ERROR(log_, "lookup: invalid authority signature");
       return Error::INVALID_SIGNATURE;
     }
 
@@ -283,6 +304,7 @@ namespace kagome::authority_discovery {
                     str2byte(signed_record.peer_signature().signature()),
                     peer_key));
     if (not peer_sig_ok) {
+      SL_ERROR(log_, "lookup: invalid peer signature");
       return Error::INVALID_SIGNATURE;
     }
 

diff --git a/core/parachain/validator/impl/parachain_processor.cpp b/core/parachain/validator/impl/parachain_processor.cpp
@@ -4504,7 +4504,7 @@ namespace kagome::parachain {
     BOOST_ASSERT(protocol);
     CHECK_OR_RET(canProcessParachains().has_value());
 
-    SL_INFO(logger_,
+    SL_DEBUG(logger_,
             "Send my view.(peer={}, protocol={})",
             peer_id,
             protocol->protocolName());
@@ -4588,7 +4588,7 @@ namespace kagome::parachain {
                 } break;
               }
             })) {
-      logger_->info("Initiated validation protocol with {}", peer_id);
+      logger_->debug("Initiated validation protocol with {}", peer_id);
     }
   }
 

diff --git a/core/telemetry/CMakeLists.txt b/core/telemetry/CMakeLists.txt
@@ -18,6 +18,7 @@ target_link_libraries(telemetry
     kagome_uri
     Boost::boost
     Boost::date_time
+    Boost::process
     OpenSSL::SSL
     RapidJSON::rapidjson
     p2p::p2p_asio_scheduler_backend

diff --git a/core/telemetry/impl/connection_impl.hpp b/core/telemetry/impl/connection_impl.hpp
@@ -12,8 +12,6 @@
 #include <memory>
 #include <queue>
 
-#include <boost/asio.hpp>
-#include <boost/asio/ssl.hpp>
 #include <boost/asio/strand.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/ssl.hpp>

diff --git a/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile b/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile
@@ -6,6 +6,7 @@ ARG ARCHITECTURE=x86_64
 
 ARG DEBIAN_VERSION=bookworm
 ARG LLVM_VERSION=19
+ARG GCC_VERSION=13
 
 FROM ${BASE_IMAGE}
 
@@ -20,6 +21,8 @@ ARG DEBIAN_VERSION
 ENV DEBIAN_VERSION=${DEBIAN_VERSION}
 ARG LLVM_VERSION
 ENV LLVM_VERSION=${LLVM_VERSION}
+ARG GCC_VERSION
+ENV GCC_VERSION=${GCC_VERSION}
 
 RUN install_packages \
         apt-transport-https \
@@ -30,6 +33,7 @@ RUN install_packages \
 RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg
 RUN echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/${DEBIAN_VERSION}/ llvm-toolchain-${DEBIAN_VERSION}-${LLVM_VERSION} main" | \
         tee -a /etc/apt/sources.list.d/llvm.list
+RUN echo "deb http://deb.debian.org/debian/ trixie main" | tee -a /etc/apt/sources.list
 
 RUN install_packages \
         build-essential \
@@ -40,8 +44,8 @@ RUN install_packages \
         llvm-${LLVM_VERSION}-dev \
         curl \
         dpkg-dev \
-        g++-12 \
-        gcc-12 \
+        g++-${GCC_VERSION} \
+        gcc-${GCC_VERSION} \
         gdb \
         gdbserver \
         git \
@@ -79,8 +83,8 @@ ENV HUNTER_PYTHON_LOCATION=/venv/bin/python3
 ENV LLVM_ROOT=/usr/lib/llvm-${LLVM_VERSION}
 ENV LLVM_DIR=/usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/
 ENV PATH=${LLVM_ROOT}/bin:${LLVM_ROOT}/share/clang:${PATH}
-ENV CC=gcc-12
-ENV CXX=g++-12
+ENV CC=gcc-${GCC_VERSION}
+ENV CXX=g++-${GCC_VERSION}
 
 RUN update-alternatives --install /usr/bin/python       python       /venv/bin/python3              90 && \
     update-alternatives --install /usr/bin/python       python       /usr/bin/python3               80 && \
@@ -90,6 +94,6 @@ RUN update-alternatives --install /usr/bin/python       python       /venv/bin/p
     update-alternatives --install /usr/bin/clang        clang        /usr/lib/llvm-${LLVM_VERSION}/bin/clang-${LLVM_VERSION}  50 && \
     update-alternatives --install /usr/bin/clang++      clang++      /usr/bin/clang++-${LLVM_VERSION}                         50 && \
     \
-    update-alternatives --install /usr/bin/gcc          gcc          /usr/bin/gcc-12                90 && \
-    update-alternatives --install /usr/bin/g++          g++          /usr/bin/g++-12                90 && \
-    update-alternatives --install /usr/bin/gcov         gcov         /usr/bin/gcov-12               90
+    update-alternatives --install /usr/bin/gcc          gcc          /usr/bin/gcc-${GCC_VERSION}    90 && \
+    update-alternatives --install /usr/bin/g++          g++          /usr/bin/g++-${GCC_VERSION}    90 && \
+    update-alternatives --install /usr/bin/gcov         gcov         /usr/bin/gcov-${GCC_VERSION}   90
diff --git a/housekeeping/docker/kagome-dev/minideb.Dockerfile b/housekeeping/docker/kagome-dev/minideb.Dockerfile
@@ -65,6 +65,11 @@ RUN --mount=type=secret,id=google_creds,target=/root/.gcp/google_creds.json \
         kagome-dev=${KAGOME_PACKAGE_VERSION} && \
         sed -i '1s/^/#/' /etc/apt/sources.list.d/kagome.list
 
+# temporary fix for libc6 (gcc-13)
+# TODO: remove when CI swithed to trixie
+RUN echo "deb http://deb.debian.org/debian/ trixie main" | tee -a /etc/apt/sources.list && apt update
+RUN apt install -y libc6 libstdc++6 libgcc-s1 -t trixie
+
 CMD ["/usr/bin/tini", "--", "/bin/bash", "-c"]
 
 

diff --git a/housekeeping/docker/kagome/minideb-release.Dockerfile b/housekeeping/docker/kagome/minideb-release.Dockerfile
@@ -11,7 +11,7 @@ RUN apt-get update && \
     wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
     add-apt-repository -y "deb http://deb.debian.org/debian/ testing main" && \
     apt-get update && \
-    apt-get install --no-install-recommends -y libgmp10 libstdc++6 libc6 libnsl2 libatomic1 gdb gdbserver && \
+    apt-get install --no-install-recommends -y libgmp10 libstdc++6 libc6 libnsl2 libatomic1 gdb gdbserver tini libncurses-dev&& \
     rm -rf /var/lib/apt/lists/*
 
 COPY kagome /usr/local/bin/
diff --git a/zombienet/docker/zombie_tester.Dockerfile b/zombienet/docker/zombie_tester.Dockerfile
@@ -128,3 +128,8 @@ RUN install_packages  \
       libseccomp2 \
       libatomic1 \
       ssh
+
+# temporary fix for libc6 (gcc-13)
+# TODO: remove when CI swithed to trixie
+RUN echo "deb http://deb.debian.org/debian/ trixie main" | tee -a /etc/apt/sources.list && apt update
+RUN apt install -y libc6 libstdc++6 libgcc-s1 -t trixie