Add periodic E2E tests against a newest Polkadot release (#147)

.github/actions/run-polkadot/action.yml

@@ -0,0 +1,46 @@
+name: Run Polkadot action
+description: Reusable action to build and run a modified version of Polkadot
+inputs:
+  polkadot-version:
+    description: 'Version tag, e.g. polkadot-v1.7.1'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/[email protected]
+      with:
+        path: substrate-tip-bot
+    - uses: actions/[email protected]
+      with:
+        repository: paritytech/polkadot-sdk
+        ref: ${{ inputs.polkadot-version }}
+        path: polkadot-sdk
+    - name: Apply Polkadot patches
+      run: git apply ../substrate-tip-bot/polkadot.e2e.patch
+      shell: bash
+      working-directory: polkadot-sdk
+    - name: Restore cached Polkadot build
+      id: polkadot-cache-restore
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          polkadot-sdk/target/release
+        key: ${{ runner.os }}-${{ inputs.polkadot-version }}-${{ hashFiles('substrate-tip-bot/polkadot.e2e.patch') }}
+    - name: Build Polkadot
+      if: steps.polkadot-cache-restore.outputs.cache-hit != 'true'
+      run: |
+        cargo build --release --locked --features=fast-runtime -p polkadot
+      shell: bash
+      working-directory: polkadot-sdk
+    - name: Save Polkadot build cache
+      uses: actions/cache/save@v3
+      with:
+        path: |
+          polkadot-sdk/target/release
+        key: ${{ runner.os }}-${{ inputs.polkadot-version }}-${{ hashFiles('substrate-tip-bot/polkadot.e2e.patch') }}
+    - name: Run a local Rococo node
+      run: |
+        polkadot-sdk/target/release/polkadot --rpc-external --no-prometheus --no-telemetry --chain=rococo-dev --tmp --alice --execution Native --rpc-port 9902 &
+        until curl -s '127.0.0.1:9902'; do sleep 3; done
+      shell: bash

.github/workflows/test-e2e-cron.yml

@@ -0,0 +1,38 @@
+name: E2E tests against most recent Polkadot release
+on:
+  schedule:
+    - cron: 0 3 * * SUN # Once a week on Sunday.
+  workflow_dispatch:
+
+jobs:
+  test-e2e-cron:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120
+    container: "${{ vars.E2E_TESTS_CONTAINER }}"
+    steps:
+      - name: Read latest polkadot tag
+        id: read-tag
+        run: |
+          # Fetch all Polkadot release tags, get the last (newest) one, and parse its name from the output.
+          TAG=$(git ls-remote --refs --tags https://github.com/paritytech/polkadot-sdk.git 'polkadot-v*' \
+            | tail -1 \
+            | sed 's,[^r]*refs/tags/,,')
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+      - name: Announce version
+        run: echo "Running tests with Polkadot version ${{ steps.read-tag.outputs.tag }}"
+      - uses: actions/[email protected]
+      - name: Start a local Polkadot node
+        uses: ./.github/actions/run-polkadot
+        with:
+          polkadot-version: "${{ steps.read-tag.outputs.tag }}"
+      - name: Wait for the node
+        run: |
+          until curl -s '127.0.0.1:9902'; do sleep 3; done
+        timeout-minutes: 1
+      - uses: actions/[email protected]
+        with:
+          node-version: "18.16"
+      - run: npm i -g [email protected]
+      - run: yarn --frozen-lockfile
+      - run: yarn test:e2e
+        timeout-minutes: 10

.github/workflows/test-e2e.yml

@@ -0,0 +1,29 @@
+name: E2E tests
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  test-e2e:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120
+    container: "${{ vars.E2E_TESTS_CONTAINER }}"
+    steps:
+      - uses: actions/[email protected]
+      - name: Start a local Polkadot node
+        uses: ./.github/actions/run-polkadot
+        with:
+          polkadot-version: polkadot-v1.7.1
+      - name: Wait for the node
+        run: |
+          until curl -s '127.0.0.1:9902'; do sleep 3; done
+        timeout-minutes: 1
+      - uses: actions/[email protected]
+        with:
+          node-version: "18.16"
+      - run: npm i -g [email protected]
+      - run: yarn --frozen-lockfile
+      - run: yarn test:e2e
+        timeout-minutes: 10

.github/workflows/test-integration.yml

@@ -0,0 +1,15 @@
+name: Integration tests
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  test-integration:
+    timeout-minutes: 15
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/[email protected]
+      - run: yarn --frozen-lockfile
+      - run: yarn test:integration --verbose

README.md

@@ -151,6 +151,15 @@ yarn test:e2e
 
 Go make a cup of tea, the tests take ~3 minutes (waiting for the various on-chain stages to pass).
 
+On CI, the E2E tests are running:
+
+- On every PR and commits pushed to `master`, against a fixed release of `polkadot-sdk`.
+- Periodically, against a most recent release of `polkadot-sdk`.
+
+The tests are running in a container specified by `E2E_TESTS_CONTAINER` [repository variable](https://docs.github.com/en/actions/learn-github-actions/variables#defining-configuration-variables-for-multiple-workflows).
+
+The container version should be kept in sync with the container used in [`polkadot-sdk` CI](https://github.com/paritytech/polkadot-sdk/blob/polkadot-v1.7.2/.github/workflows/fmt-check.yml#L18).
+
 ## Contributing
 
 If you have suggestions for how substrate-tip-bot could be improved, or want to report a bug, open