diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml new file mode 100644 index 0000000000000..058c4801f7df8 --- /dev/null +++ b/.github/workflows/build-container.yml @@ -0,0 +1,54 @@ +name: Build container + +on: + workflow_call: + outputs: + path: + description: "Path to built container" + value: ghcr.io/${{ jobs.build.outputs.repo }}/dashcore-ci-runner:${{ jobs.build.outputs.tag }} + +env: + DOCKER_DRIVER: overlay2 + +jobs: + build: + name: Build container + runs-on: ubuntu-22.04 + outputs: + tag: ${{ steps.prepare.outputs.tag }} + repo: ${{ steps.prepare.outputs.repo }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Prepare variables + id: prepare + run: | + BRANCH_NAME=$(echo "${GITHUB_REF##*/}" | tr '[:upper:]' '[:lower:]') + REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + echo "tag=${BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "repo=${REPO_NAME}" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: ./contrib/containers/ci + file: ./contrib/containers/ci/Dockerfile + push: true + tags: | + ghcr.io/${{ steps.prepare.outputs.repo }}/dashcore-ci-runner:${{ steps.prepare.outputs.tag }} + ghcr.io/${{ steps.prepare.outputs.repo }}/dashcore-ci-runner:latest + cache-from: type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo }}/dashcore-ci-runner:latest + cache-to: type=inline diff --git a/.github/workflows/build-depends.yml b/.github/workflows/build-depends.yml new file mode 100644 index 0000000000000..f77e35c366a9c --- /dev/null +++ b/.github/workflows/build-depends.yml @@ -0,0 +1,79 @@ +name: Build depends + +on: + workflow_call: + inputs: + build-target: + description: "Target name as defined by matrix.sh" + required: true + type: string + container-path: + description: "Path to built container at registry" + required: true + type: string + outputs: + key: + description: "Key needed for restoring depends cache" + value: ${{ jobs.build-depends.outputs.key }} + +jobs: + build-depends: + name: Build depends + runs-on: ubuntu-22.04 + outputs: + key: ${{ steps.restore.outputs.cache-primary-key }} + container: + image: ${{ inputs.container-path }} + options: --user root + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Initial setup + id: setup + run: | + BUILD_TARGET="${{ inputs.build-target }}" + source ./ci/dash/matrix.sh + echo "DEP_OPTS=${DEP_OPTS}" >> "${GITHUB_OUTPUT}" + echo "HOST=${HOST}" >> "${GITHUB_OUTPUT}" + DEP_HASH="$(echo -n "${BUILD_TARGET}" "${DEP_OPTS}" "${HOST}" | sha256sum | head -c 64)" + echo "\"${BUILD_TARGET}\" has HOST=\"${HOST}\" and DEP_OPTS=\"${DEP_OPTS}\" with hash \"${DEP_HASH}\"" + echo "DEP_HASH=${DEP_HASH}" >> "${GITHUB_OUTPUT}" + + shell: bash + + - name: Cache depends sources + uses: actions/cache@v4 + with: + path: | + depends/sources + key: depends-sources-${{ hashFiles('depends/packages/*') }} + restore-keys: | + depends-sources-${{ hashFiles('depends/packages/*') }} + depends-sources- + + - name: Restore cached depends + uses: actions/cache/restore@v4 + id: restore + with: + path: | + depends/built + depends/${{ steps.setup.outputs.HOST }} + key: ${{ runner.os }}-depends-${{ inputs.build-target }}-${{ steps.setup.outputs.DEP_HASH }}-${{ hashFiles('depends/packages/*') }} + restore-keys: | + ${{ runner.os }}-depends-${{ inputs.build-target }}-${{ hashFiles('depends/packages/*') }} + ${{ runner.os }}-depends-${{ inputs.build-target }} + + - name: Build depends + run: env ${{ steps.setup.outputs.DEP_OPTS }} HOST=${{ steps.setup.outputs.HOST }} make -j$(nproc) -C depends + + - name: Save depends cache + uses: actions/cache/save@v4 + if: steps.restore.outputs.cache-hit != 'true' + with: + path: | + depends/built + depends/${{ steps.setup.outputs.HOST }} + key: ${{ steps.restore.outputs.cache-primary-key }} diff --git a/.github/workflows/build-src.yml b/.github/workflows/build-src.yml new file mode 100644 index 0000000000000..5fd76a580187d --- /dev/null +++ b/.github/workflows/build-src.yml @@ -0,0 +1,84 @@ +name: Build source + +on: + workflow_call: + inputs: + build-target: + description: "Target name as defined by inputs.sh" + required: true + type: string + container-path: + description: "Path to built container at registry" + required: true + type: string + depends-key: + description: "Key needed to access cached depends" + required: true + type: string + +jobs: + build-src: + name: Build source + runs-on: ubuntu-22.04 + container: + image: ${{ inputs.container-path }} + options: --user root + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Initial setup + id: setup + run: | + git config --global --add advice.detachedHead false + git config --global --add safe.directory "$PWD" + GIT_HEAD="$(git rev-parse HEAD)" + git checkout develop + git checkout "${GIT_HEAD}" + BUILD_TARGET="${{ inputs.build-target }}" + source ./ci/dash/matrix.sh + echo "HOST=${HOST}" >> $GITHUB_OUTPUT + echo "PR_BASE_SHA=${{ github.event.pull_request.base.sha || '' }}" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore depends cache + uses: actions/cache/restore@v4 + with: + path: | + depends/built + depends/${{ steps.setup.outputs.HOST }} + key: ${{ inputs.depends-key }} + fail-on-cache-miss: true + + - name: Manage ccache + uses: actions/cache@v4 + with: + path: | + /cache + key: ${{ runner.os }}-${{ inputs.build-target }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ inputs.build-target }}-${{ github.sha }} + ${{ runner.os }}-${{ inputs.build-target }}-${{ steps.setup.outputs.HOST }} + ${{ runner.os }}-${{ inputs.build-target }} + + - name: Build source and run unit tests + run: | + CCACHE_SIZE="400M" + CACHE_DIR="/cache" + mkdir /output + BASE_OUTDIR="/output" + BUILD_TARGET="${{ inputs.build-target }}" + source ./ci/dash/matrix.sh + ./ci/dash/build_src.sh + ./ci/dash/test_unittests.sh + shell: bash + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts-${{ inputs.build-target }} + path: | + /output diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8d3889e47608..95e509d0e086a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,184 +13,136 @@ env: FAST_MODE: false jobs: - build-image: - name: Build Image - runs-on: ubuntu-22.04 - outputs: - image-tag: ${{ steps.prepare.outputs.image-tag }} - repo-name: ${{ steps.prepare.outputs.repo-name }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Prepare - id: prepare - run: | - BRANCH_NAME=$(echo "${GITHUB_REF##*/}" | tr '[:upper:]' '[:lower:]') - REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') - echo "image-tag=${BRANCH_NAME}" >> $GITHUB_OUTPUT - echo "repo-name=${REPO_NAME}" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v6 - with: - context: ./contrib/containers/ci - file: ./contrib/containers/ci/Dockerfile - push: true - tags: | - ghcr.io/${{ steps.prepare.outputs.repo-name }}/dashcore-ci-runner:${{ steps.prepare.outputs.image-tag }} - ghcr.io/${{ steps.prepare.outputs.repo-name }}/dashcore-ci-runner:latest - cache-from: type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo-name }}/dashcore-ci-runner:latest - cache-to: type=inline - - build-depends: - name: Build Dependencies - needs: build-image - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - include: - - build_target: arm-linux - host: arm-linux-gnueabihf - - build_target: linux64 - host: x86_64-pc-linux-gnu - - build_target: win64 - host: x86_64-w64-mingw32 - - container: - image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} - options: --user root - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Cache depends sources - uses: actions/cache@v4 - with: - path: | - depends/sources - key: depends-sources-${{ hashFiles('depends/packages/*') }} - restore-keys: | - depends-sources- - - - name: Cache depends - uses: actions/cache@v4 - with: - path: | - depends/built - depends/${{ matrix.host }} - key: ${{ runner.os }}-depends-${{ matrix.build_target }}-${{ hashFiles('depends/packages/*') }} - restore-keys: | - ${{ runner.os }}-depends-${{ matrix.build_target }}-${{ hashFiles('depends/packages/*') }} - ${{ runner.os }}-depends-${{ matrix.build_target }} - - - name: Build depends - run: make -j$(nproc) -C depends HOST=${{ matrix.host }} - - build: - name: Build - needs: [build-image, build-depends] - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - include: - - build_target: arm-linux - host: arm-linux-gnueabihf - depends_on: arm-linux - - build_target: linux64 - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_cxx20 - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_fuzz - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_nowallet - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_sqlite - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_tsan - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_ubsan - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: win64 - host: x86_64-w64-mingw32 - depends_on: win64 - container: - image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} - options: --user root - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - - name: Restore depends cache - uses: actions/cache/restore@v4 - with: - path: | - depends/built - depends/${{ matrix.host }} - key: ${{ runner.os }}-depends-${{ matrix.depends_on }}-${{ hashFiles('depends/packages/*') }} - - - name: Determine PR Base SHA - id: vars - run: | - echo "PR_BASE_SHA=${{ github.event.pull_request.base.sha || '' }}" >> $GITHUB_OUTPUT - - - name: CCache - uses: actions/cache@v4 - with: - path: | - /cache - key: ${{ runner.os }}-${{ matrix.build_target }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ matrix.build_target }}-${{ github.sha }} - ${{ runner.os }}-${{ matrix.build_target }}-${{ steps.vars.outputs.PR_BASE_SHA }} - ${{ runner.os }}-${{ matrix.build_target }} - - - name: Build source and run tests - run: | - git config --global --add advice.detachedHead false - git config --global --add safe.directory "$PWD" - GIT_HEAD="$(git rev-parse HEAD)" - git checkout develop - git checkout ${GIT_HEAD} - CCACHE_SIZE="400M" - CACHE_DIR="/cache" - mkdir /output - BASE_OUTDIR="/output" - BUILD_TARGET="${{ matrix.build_target }}" - source ./ci/dash/matrix.sh - ./ci/dash/build_src.sh - ./ci/dash/test_unittests.sh - shell: bash - - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: build-artifacts-${{ matrix.build_target }} - path: | - /output + container: + name: Build container + uses: ./.github/workflows/build-container.yml + + depends-arm-linux: + name: arm-linux-gnueabihf + uses: ./.github/workflows/build-depends.yml + needs: [container] + with: + build-target: arm-linux + container-path: ${{ needs.container.outputs.path }} + + depends-linux64: + name: x86_64-pc-linux-gnu + uses: ./.github/workflows/build-depends.yml + needs: [container] + with: + build-target: linux64 + container-path: ${{ needs.container.outputs.path }} + + depends-linux64_multiprocess: + name: x86_64-pc-linux-gnu_multiprocess + uses: ./.github/workflows/build-depends.yml + needs: [container] + with: + build-target: linux64_multiprocess + container-path: ${{ needs.container.outputs.path }} + + depends-linux64_nowallet: + name: x86_64-pc-linux-gnu_nowallet + uses: ./.github/workflows/build-depends.yml + needs: [container] + with: + build-target: linux64_nowallet + container-path: ${{ needs.container.outputs.path }} + + depends-win64: + name: x86_64-w64-mingw32 + uses: ./.github/workflows/build-depends.yml + needs: [container] + with: + build-target: win64 + container-path: ${{ needs.container.outputs.path }} + + src-arm-linux: + name: arm-linux-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-arm-linux] + with: + build-target: arm-linux + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-arm-linux.outputs.key }} + + src-linux64: + name: linux64-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-linux64] + with: + build-target: linux64 + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-linux64.outputs.key }} + + src-linux64_cxx20: + name: linux64_cxx20-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-linux64] + with: + build-target: linux64_cxx20 + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-linux64.outputs.key }} + + src-linux64_fuzz: + name: linux64_fuzz-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-linux64] + with: + build-target: linux64_fuzz + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-linux64.outputs.key }} + + src-linux64_multiprocess: + name: linux64_multiprocess-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-linux64_multiprocess] + with: + build-target: linux64_multiprocess + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-linux64_multiprocess.outputs.key }} + + src-linux64_nowallet: + name: linux64_nowallet-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-linux64_nowallet] + with: + build-target: linux64_nowallet + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-linux64_nowallet.outputs.key }} + + src-linux64_sqlite: + name: linux64_sqlite-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-linux64] + with: + build-target: linux64_sqlite + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-linux64.outputs.key }} + + src-linux64_tsan: + name: linux64_tsan-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-linux64_multiprocess] + with: + build-target: linux64_tsan + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-linux64_multiprocess.outputs.key }} + + src-linux64_ubsan: + name: linux64_ubsan-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-linux64] + with: + build-target: linux64_ubsan + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-linux64.outputs.key }} + + src-win64: + name: win64-build + uses: ./.github/workflows/build-src.yml + needs: [container, depends-win64] + with: + build-target: win64 + container-path: ${{ needs.container.outputs.path }} + depends-key: ${{ needs.depends-win64.outputs.key }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30f47434c478b..99c281ebe47f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,10 @@ builder-image: image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG before_script: - | - echo HOST=${HOST} + echo BUILD_TARGET="${BUILD_TARGET}" + source ./ci/dash/matrix.sh + echo HOST="${HOST}" + echo DEP_OPTS="${DEP_OPTS}" if [[ "${HOST}" == "x86_64-apple-darwin" ]]; then ./contrib/containers/guix/scripts/setup-sdk fi @@ -73,6 +76,8 @@ builder-image: - export CACHE_DIR=$CI_PROJECT_DIR/cache - echo BUILD_TARGET=$BUILD_TARGET - source ./ci/dash/matrix.sh + - echo HOST=${HOST} + - echo DEP_OPTS=${DEP_OPTS} # Setup some environment variables - | @@ -170,43 +175,40 @@ builder-image: arm-linux-gnueabihf: extends: .build-depends-template variables: - HOST: arm-linux-gnueabihf + BUILD_TARGET: arm-linux x86_64-w64-mingw32: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-w64-mingw32 + BUILD_TARGET: win64 -x86_64-pc-linux-gnu_debug: +x86_64-pc-linux-gnu: extends: .build-depends-template variables: - HOST: x86_64-pc-linux-gnu - DEP_OPTS: "DEBUG=1" + BUILD_TARGET: linux64 x86_64-pc-linux-gnu_nowallet: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-pc-linux-gnu - DEP_OPTS: "NO_WALLET=1" + BUILD_TARGET: linux64_nowallet x86_64-pc-linux-gnu_multiprocess: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-pc-linux-gnu - DEP_OPTS: "DEBUG=1 MULTIPROCESS=1" + BUILD_TARGET: linux64_multiprocess x86_64-apple-darwin: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-apple-darwin + BUILD_TARGET: mac ### @@ -229,7 +231,7 @@ win64-build: linux64-build: extends: .build-template needs: - - x86_64-pc-linux-gnu_debug + - x86_64-pc-linux-gnu variables: BUILD_TARGET: linux64 @@ -238,7 +240,7 @@ linux64_cxx20-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu_debug + - x86_64-pc-linux-gnu variables: BUILD_TARGET: linux64_cxx20 @@ -247,7 +249,7 @@ linux64_sqlite-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu_debug + - x86_64-pc-linux-gnu variables: BUILD_TARGET: linux64_sqlite @@ -256,7 +258,7 @@ linux64_fuzz-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu_debug + - x86_64-pc-linux-gnu variables: BUILD_TARGET: linux64_fuzz @@ -265,7 +267,7 @@ linux64_fuzz-build: # - .build-template # - .skip-in-fast-mode-template # needs: -# - x86_64-pc-linux-gnu_debug +# - x86_64-pc-linux-gnu # variables: # BUILD_TARGET: linux64_asan @@ -274,7 +276,7 @@ linux64_tsan-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu_debug + - x86_64-pc-linux-gnu_multiprocess variables: BUILD_TARGET: linux64_tsan @@ -283,7 +285,7 @@ linux64_ubsan-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu_debug + - x86_64-pc-linux-gnu variables: BUILD_TARGET: linux64_ubsan @@ -310,7 +312,7 @@ linux64_multiprocess-build: # - .build-template # - .skip-in-fast-mode-template # needs: -# - x86_64-pc-linux-gnu_debug +# - x86_64-pc-linux-gnu # variables: # BUILD_TARGET: linux64_valgrind diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index 04d1b1df55f99..91e70a4ca3b64 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -7,8 +7,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_multiprocess +export HOST=x86_64-pc-linux-gnu export PACKAGES="cmake python3 llvm clang" -export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" +export DEP_OPTS="MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" export GOAL="install" export TEST_RUNNER_EXTRA="--v2transport" export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang-18 CXX=clang++-18" # Use clang to avoid OOM diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 1d28e24152308..8794285297b84 100755 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -7,6 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet +export HOST=x86_64-pc-linux-gnu export PACKAGES="python3-zmq" export DEP_OPTS="NO_WALLET=1" export GOAL="install" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index bf369f153845c..0151cf4e8a1f0 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -7,8 +7,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 +export HOST=x86_64-pc-linux-gnu export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" -export DEP_OPTS="NO_UPNP=1 DEBUG=1" +export DEP_OPTS="" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_pruning,feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false"