Script to run full set of backwards compatibility tests

e2e/Makefile

@@ -10,4 +10,7 @@ cleanup-ibc-test-containers:
 e2e-test: cleanup-ibc-test-containers
 	./scripts/run-e2e.sh $(suite) $(test)
 
-.PHONY: cleanup-ibc-test-containers e2e-test
+compatibility-tests:
+	./scripts/run-compatibility-tests.sh $(version)
+
+.PHONY: cleanup-ibc-test-containers e2e-test compatibility-tests

e2e/README.md

@@ -16,7 +16,8 @@
    - a. [ibctest](#ibctest)
    - b. [CI configuration](#ci-configuration)
 3. [Github Workflows](#github-workflows) 
-4. [Troubleshooting](#troubleshooting)
+4. [Running Compatibility Tests](#running-compatibility-tests)
+5. [Troubleshooting](#troubleshooting)
 
 
 ## How to write tests
@@ -39,11 +40,11 @@ Tests can be run using a Makefile target under the e2e directory. `e2e/Makefile`
 
 There are several envinronment variables that alter the behaviour of the make target.
 
-| Environment Variable      | Description | Default Value|
-| ----------- | ----------- | ----------- |
-| SIMD_IMAGE      | The image that will be used for simd       | ibc-go-simd |
-| SIMD_TAG   | The tag used for simd        | latest|
-| RLY_TAG   | The tag used for the go relayer        | main|
+| Environment Variable      | Description                               | Default Value|
+| ----------- |-------------------------------------------| ----------- |
+| CHAIN_IMAGE      | The image that will be used for the chain | ibc-go-simd |
+| CHAIN_A_TAG   | The tag used for the chain                | latest|
+| RLY_TAG   | The tag used for the go relayer           | main|
 
 
 > Note: when running tests locally, **no images are pushed** to the `ghcr.io/cosmos/ibc-go-simd` registry.
@@ -329,6 +330,23 @@ Alternatively, the [gh](https://cli.github.com/) CLI tool can be used to trigger
 gh workflow run "Build Simd Image" -f tag=v3.0.0
 ```
 
+### Running Compatibility Tests
+
+A full matrix of tests can be configured in json format. See [this file](./scripts/test-matricies/main/test-matrix.json) as a reference.
+
+To run all of the tests specified in this file, run
+
+```bash
+make compatibility-tests
+```
+
+This will run a GitHub Action for each entry, and display markdown which can be used in GitHub issue
+bodies to provide links to each of the workflows.
+
+Note: a version field which corresponds to a subdirectory under scripts/test-matricies can also be specified. Once a file exists there,
+e.g. `scripts/test-matricies/v5.0.0/test-matrix.json`, that file will be used to provide values to the GitHub workflows
+if we run `make compatibility-tests version=v5.0.0`
+
 ### Troubleshooting
 
 * On Mac, after running a lot of tests, it can happen that containers start failing. To fix this, you can try clearing existing containers and restarting the docker daemon.

e2e/scripts/run-compatibility-tests.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -Eeou pipefail
+
+# run_gh_workflow runs a single github workflow and outputs the workflow information in a markdown format.
+function run_gh_workflow(){
+        local test_entry_point="${1}"
+        local chain_binary="${2}"
+        local chain_a_tag="${3}"
+        local chain_b_tag="${4}"
+
+        # any changes to the workflows on this branch will be used.
+        local current_branch="$(git branch --show-current)"
+
+        # manually trigger a workflow using each entry from the list
+        gh workflow run "e2e-manual-${chain_binary}.yaml" --ref="${current_branch}" \
+          -f chain-a-tag="${chain_a_tag}" \
+          -f chain-b-tag="${chain_b_tag}" \
+          -f test-entry-point="${test_entry_point}" > /dev/null
+        # it takes some time for the test to appear in the list, we need to wait for it to show up.
+        sleep 2
+        # this assumes nobody else has run a manual workflow in the last 2 seconds
+        run_id="$(gh run list "--workflow=e2e-manual-${chain_binary}.yaml" | grep workflow_dispatch | grep -Eo "[0-9]{9,11}" | head -n 1)"
+        echo "  - [ ] [chain A (${chain_a_tag}) -> chain B (${chain_b_tag})](https://github.com/cosmos/ibc-go/actions/runs/${run_id})"
+}
+
+# run_full_compatibility_suite runs all tests specified in the test-matrix.json file.
+function run_full_compatibility_suite(){
+    local matrix_version="${1}"
+    local matrix_file_path="${2:-"scripts/test-matricies/${matrix_version}/test-matrix.json"}"
+
+    echo "## Backwards compatibility tests"
+    echo "### Matrix Version: ${matrix_version}"
+
+    jq -c -r '.[]' "${matrix_file_path}" | while read arguments; do
+        test_entry_point="$(echo ${arguments} | jq -r -c '."test-entry-point"')"
+        test_arguments="$(echo ${arguments} | jq -r -c '."tests"')"
+        chain_binary="$(echo ${arguments} | jq -r -c '."chain-binary"')"
+        echo "#### ${test_entry_point}"
+        echo ${test_arguments} | jq -c -r '.[]' | while read test; do
+            chain_a_tag="$(echo ${test} | jq -r -c '."chain-a-tag"')"
+            chain_b_tag="$(echo ${test} | jq -r -c '."chain-b-tag"')"
+            run_gh_workflow "${test_entry_point}" "${chain_binary}" "${chain_a_tag}" "${chain_b_tag}"
+        done
+    done
+}
+
+VERSION_MATRIX="${1:-main}"
+run_full_compatibility_suite "${VERSION_MATRIX}"

e2e/scripts/test-matricies/main/test-matrix.json

@@ -0,0 +1,68 @@
+[
+  {
+    "test-entry-point": "TestTransferTestSuite",
+    "chain-binary": "simd",
+    "tests": [
+      {
+        "chain-a-tag": "main",
+        "chain-b-tag": "v4.0.0"
+      },
+      {
+        "chain-a-tag": "main",
+        "chain-b-tag": "v3.0.0"
+      },
+      {
+        "chain-a-tag": "main",
+        "chain-b-tag": "v2.0.0"
+      },
+      {
+        "chain-a-tag": "v4.0.0",
+        "chain-b-tag": "main"
+      },
+      {
+        "chain-a-tag": "v3.0.0",
+        "chain-b-tag": "main"
+      },
+      {
+        "chain-a-tag": "v2.0.0",
+        "chain-b-tag": "main"
+      }
+    ]
+  },
+  {
+    "test-entry-point": "TestFeeMiddlewareTestSuite",
+    "chain-binary": "simd",
+    "tests": [
+      {
+        "chain-a-tag": "main",
+        "chain-b-tag": "v4.0.0"
+      },
+      {
+        "chain-a-tag": "v4.0.0",
+        "chain-b-tag": "main"
+      }
+    ]
+  },
+  {
+    "test-entry-point": "TestInterchainAccountsTestSuite",
+    "chain-binary": "icad",
+    "tests": [
+      {
+        "chain-a-tag": "v0.3.2",
+        "chain-b-tag": "v0.1.3"
+      },
+      {
+        "chain-a-tag": "v0.1.3",
+        "chain-b-tag": "v0.3.2"
+      },
+      {
+        "chain-a-tag": "v0.2.2",
+        "chain-b-tag": "v0.3.2"
+      },
+      {
+        "chain-a-tag": "v0.3.2",
+        "chain-b-tag": "v0.2.2"
+      }
+    ]
+  }
+]