CI: Add Ecosystem tests #3183
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Tests" | ||
| # If you modify more test jobs, ensure that you add them as required to the job "confirmTestPassed" | ||
| # which is located at the end of this file (more info in the job) | ||
| on: | ||
| push: | ||
| branches: ["main", "release-*"] | ||
| pull_request: | ||
| workflow_dispatch: | ||
| env: | ||
| FRAME_OMNI_BENCHER_RELEASE_VERSION: polkadot-stable2409 | ||
| # cancel previous runs | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | ||
| cancel-in-progress: true | ||
| jobs: | ||
| # This generates a matrix with all the required jobs which will be run in the next step | ||
| runtime-matrix: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| runtime: ${{ steps.runtime.outputs.runtime }} | ||
| name: Extract runtimes from matrix | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - id: runtime | ||
| run: | | ||
| TASKS=$(echo $(cat .github/workflows/runtimes-matrix.json) | sed 's/ //g' ) | ||
| echo $TASKS | ||
| echo "runtime=$TASKS" >> $GITHUB_OUTPUT | ||
| integration-test-matrix: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| itest: ${{ steps.itest.outputs.itest }} | ||
| name: Extract integration tests from matrix | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - id: itest | ||
| run: | | ||
| TASKS=$(echo $(cat .github/workflows/integration-tests-matrix.json) | sed 's/ //g' ) | ||
| echo $TASKS | ||
| echo "itest=$TASKS" >> $GITHUB_OUTPUT | ||
| # Job required by "confirmTestPassed" | ||
| runtime-test: | ||
| needs: [runtime-matrix] | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| matrix: | ||
| runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Install updates and dependencies | ||
| run: .github/install-deps.sh | ||
| - name: Set rust version via common env file | ||
| run: cat .github/env >> $GITHUB_ENV | ||
| - name: Install stable toolchain | ||
| uses: dtolnay/rust-toolchain@master | ||
| with: | ||
| targets: "wasm32-unknown-unknown" | ||
| components: "rust-src" | ||
| toolchain: "${{env.RUST_STABLE_VERSION}}" | ||
| - name: Fetch cache | ||
| uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7 | ||
| with: | ||
| shared-key: "fellowship-cache-runtime-tests" | ||
| save-if: ${{ github.ref == 'refs/heads/main' }} | ||
| - name: Download frame-omni-bencher | ||
| run: | | ||
| curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/$FRAME_OMNI_BENCHER_RELEASE_VERSION/frame-omni-bencher -o frame-omni-bencher | ||
| chmod +x ./frame-omni-bencher | ||
| ./frame-omni-bencher --version | ||
| shell: bash | ||
| - name: Test ${{ matrix.runtime.name }} | ||
| run: cargo test -p ${{ matrix.runtime.package }} --release --locked -q | ||
| env: | ||
| RUSTFLAGS: "-C debug-assertions -D warnings" | ||
| - name: Test all features ${{ matrix.runtime.name }} | ||
| run: cargo test -p ${{ matrix.runtime.package }} --release --locked -q --all-features | ||
| env: | ||
| RUSTFLAGS: "-C debug-assertions -D warnings" | ||
| SKIP_WASM_BUILD: 1 | ||
| WS: ${{ matrix.runtime.uri }} | ||
| RUST_LOG: "remote-ext=info" | ||
| - name: Test benchmarks ${{ matrix.runtime.name }} | ||
| run: | | ||
| PACKAGE_NAME=${{ matrix.runtime.package }} | ||
| RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm | ||
| RUNTIME_BLOB_PATH=./target/production/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME | ||
| # build wasm | ||
| echo "Preparing wasm for benchmarking RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH" | ||
| cargo build --profile production -p ${{ matrix.runtime.package }} --features=runtime-benchmarks -q --locked | ||
| # run benchmarking | ||
| echo "Running benchmarking for RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH" | ||
| ./frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1 | ||
| env: | ||
| RUSTFLAGS: "-C debug-assertions -D warnings" | ||
| # Job required by "confirmTestPassed" | ||
| integration-test: | ||
| needs: [integration-test-matrix] | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| matrix: | ||
| itest: ${{ fromJSON(needs.integration-test-matrix.outputs.itest) }} | ||
| steps: | ||
| - name: Cancel previous runs | ||
| uses: styfle/cancel-workflow-action@b173b6ec0100793626c2d9e6b90435061f4fc3e5 # v0.11.0 | ||
| with: | ||
| access_token: ${{ github.token }} | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Install updates and dependencies | ||
| run: .github/install-deps.sh | ||
| - name: Set rust version via common env file | ||
| run: cat .github/env >> $GITHUB_ENV | ||
| - name: Install stable toolchain | ||
| uses: dtolnay/rust-toolchain@master | ||
| with: | ||
| targets: "wasm32-unknown-unknown" | ||
| components: "rust-src" | ||
| toolchain: "${{env.RUST_STABLE_VERSION}}" | ||
| - name: Fetch cache | ||
| uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7 | ||
| with: | ||
| shared-key: "fellowship-cache-integration-tests" | ||
| save-if: ${{ github.ref == 'refs/heads/main' }} | ||
| - name: Test ${{ matrix.itest.name }} | ||
| run: cargo test -p ${{ matrix.itest.package }} --release --locked -q | ||
| env: | ||
| RUSTFLAGS: "-C debug-assertions -D warnings" | ||
| # Job required by "confirmTestPassed" | ||
| build-chain-spec-generator: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Cancel previous runs | ||
| uses: styfle/cancel-workflow-action@b173b6ec0100793626c2d9e6b90435061f4fc3e5 # v0.11.0 | ||
| with: | ||
| access_token: ${{ github.token }} | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Install updates and dependencies | ||
| run: .github/install-deps.sh | ||
| - name: Set rust version via common env file | ||
| run: cat .github/env >> $GITHUB_ENV | ||
| - name: Install stable toolchain | ||
| uses: dtolnay/rust-toolchain@master | ||
| with: | ||
| targets: "wasm32-unknown-unknown" | ||
| components: "rust-src" | ||
| toolchain: "${{env.RUST_STABLE_VERSION}}" | ||
| - name: Fetch cache | ||
| uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7 | ||
| with: | ||
| shared-key: "fellowship-cache-chain-spec-tests" | ||
| save-if: ${{ github.ref == 'refs/heads/main' }} | ||
| - name: Build | ||
| run: cargo test -p chain-spec-generator --release --locked -q --features=runtime-benchmarks | ||
| env: | ||
| RUSTFLAGS: "-C debug-assertions -D warnings" | ||
| SKIP_WASM_BUILD: 1 | ||
| # Job required by "confirmTestPassed" | ||
| zombienet-smoke: | ||
| needs: [build-chain-spec-generator] | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Cancel previous runs | ||
| uses: styfle/cancel-workflow-action@b173b6ec0100793626c2d9e6b90435061f4fc3e5 # v0.11.0 | ||
| with: | ||
| access_token: ${{ github.token }} | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Install updates and dependencies | ||
| run: .github/install-deps.sh | ||
| - name: Set rust version via common env file | ||
| run: cat .github/env >> $GITHUB_ENV | ||
| - name: Install stable toolchain | ||
| uses: dtolnay/rust-toolchain@master | ||
| with: | ||
| targets: "wasm32-unknown-unknown" | ||
| components: "rust-src" | ||
| toolchain: "${{env.RUST_STABLE_VERSION}}" | ||
| - name: Fetch cache | ||
| uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7 | ||
| with: | ||
| shared-key: "fellowship-cache-zombienet-tests" | ||
| save-if: ${{ github.ref == 'refs/heads/main' }} | ||
| - name: Build | ||
| run: | | ||
| cargo build -p chain-spec-generator --features fast-runtime --release --locked | ||
| - name: Zombienet smoke test | ||
| timeout-minutes: 20 | ||
| run: | | ||
| export PATH=$(pwd)/target/release:$PATH | ||
| cargo test --manifest-path integration-tests/zombienet/Cargo.toml | ||
| build-runtimes: | ||
| needs: [ runtime-matrix ] | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| # Ensure the other jobs are continue | ||
| fail-fast: false | ||
| matrix: | ||
| runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }} | ||
| steps: | ||
| - name: Checkout sources | ||
| uses: actions/checkout@v4 | ||
| - name: Set rust version via common env file | ||
| run: cat .github/env >> $GITHUB_ENV | ||
| - name: Install stable toolchain | ||
| run: | | ||
| rustup install $RUST_STABLE_VERSION | ||
| rustup default $RUST_STABLE_VERSION | ||
| rustup target add wasm32-unknown-unknown | ||
| rustup component add rust-src | ||
| - name: Fetch cache | ||
| uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2.7.0 | ||
| with: | ||
| shared-key: "fellowship-cache-build-runtimes-test" | ||
| - name: Build runtime ${{ matrix.runtime.name }} | ||
| run: | | ||
| PACKAGE_NAME=${{ matrix.runtime.package }} | ||
| RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm | ||
| RUNTIME_BLOB_PATH=./target/production/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME | ||
| # Build wasm | ||
| echo "Building wasm RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH" | ||
| # Find out if the `metadata-hash` feature exists for the given package. | ||
| if cargo metadata --format-version=1 | jq '.packages[] | select(.name=="${{ matrix.runtime.package }}") | .features' | grep metadata-hash; then | ||
| FEATURES="--features=metadata-hash" | ||
| fi | ||
| # We only enable `metadata-hash`, but not `on-chain-release-build` to still have logs enabled. | ||
| echo "Setting features: ${FEATURES}" | ||
| cargo build --profile production -p ${{ matrix.runtime.package }} $FEATURES -q --locked | ||
| echo "RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH" >> $GITHUB_ENV | ||
| - name: Upload ${{ matrix.runtime.name }} | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: ${{ matrix.runtime.name }} | ||
| path: ${{ env.RUNTIME_BLOB_PATH }} | ||
| ecosystem-tests: | ||
| needs: [runtime-matrix, build-runtimes] | ||
| runs-on: ubuntu-22.04 | ||
| if: github.event_name == 'pull_request' | ||
| steps: | ||
| - name: Checkout sources | ||
| uses: actions/checkout@v4 | ||
| - name: Checkout polkadot-ecosystem-tests | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: open-web3-stack/polkadot-ecosystem-tests | ||
| path: ecosystem-tests | ||
| - name: Download WASM artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| merge-multiple: true | ||
| - name: Setup runtime overrides | ||
| run: | | ||
| jq --raw-output '.[] | .name, .package' .github/workflows/runtimes-matrix.json | while read -r NAME && read -r PACKAGE; do | ||
| NAME="$(echo $NAME | tr -d '-' | tr '[:lower:]' '[:upper:]')" | ||
| RUNTIME_BLOB_NAME=$(echo $PACKAGE | sed 's/-/_/g').compact.compressed.wasm | ||
| echo "Setting runtime override ${NAME}_WASM=$(pwd)/${RUNTIME_BLOB_NAME}" | ||
| # Set the path to the build runtimes | ||
| eval "export ${NAME}_WASM=$(pwd)/${RUNTIME_BLOB_NAME}" | ||
| done | ||
| - name: Installing dependencies of ecosystem tests | ||
| working-directory: ./ecosystem-tests | ||
| run: yarn install | ||
| - name: Running ecosystem tests | ||
| working-directory: ./ecosystem-tests | ||
| uses: nick-fields/retry@v3 | ||
|
Check failure on line 317 in .github/workflows/test.yml
|
||
| with: | ||
| timeout_minutes: 60 | ||
| max_attempts: 3 | ||
| command: cd ecosystem-tests && yarn test | ||
| # This will only run if all the tests in its "needs" array passed. | ||
| # Add this as your required job, becuase if the matrix changes size (new things get added) | ||
| # it will still require all the steps to succeed. | ||
| # If you add more jobs, remember to add them to the "needs" array. | ||
| confirmTestPassed: | ||
| runs-on: ubuntu-latest | ||
| name: All tests passed | ||
| # If any new job gets added, be sure to add it to this list | ||
| needs: | ||
| - runtime-test | ||
| - integration-test | ||
| - build-chain-spec-generator | ||
| - zombienet-smoke | ||
| - ecosystem-tests | ||
| steps: | ||
| - run: echo '### Good job! All the tests passed 🚀' >> $GITHUB_STEP_SUMMARY | ||
