From e31ef72ae09cf1f4ce0787f2c649c0169bf51a00 Mon Sep 17 00:00:00 2001 From: Mak Date: Thu, 21 Sep 2023 18:02:49 +0100 Subject: [PATCH 1/9] update contributing guide and ui-tests scripts --- .gitlab/pipeline/test.yml | 4 +-- docs/CONTRIBUTING.md | 15 +++++++- polkadot/scripts/update-rust-stable.sh | 39 --------------------- polkadot/scripts/update-ui-tests.sh | 5 +++ scripts/update-ui-tests.sh | 28 +++++++++++++++ substrate/.maintain/update-rust-stable.sh | 42 ----------------------- substrate/.maintain/update-ui-tests.sh | 8 +++++ 7 files changed, 57 insertions(+), 84 deletions(-) delete mode 100755 polkadot/scripts/update-rust-stable.sh create mode 100644 polkadot/scripts/update-ui-tests.sh create mode 100644 scripts/update-ui-tests.sh delete mode 100755 substrate/.maintain/update-rust-stable.sh create mode 100644 substrate/.maintain/update-ui-tests.sh diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 3a4f5e71247c..ad0ef4d9b8e6 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -314,8 +314,8 @@ node-bench-regression-guard: --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA" after_script: [""] -# if this fails (especially after rust version upgrade) run -# ./substrate/.maintain/update-rust-stable.sh +# if this fails run `bot update-ui` in the Pull Request or "./scripts/update-ui-tests.sh" locally +# see ./docs/CONTRIBUTING.md#ui-tests test-frame-ui: stage: test extends: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index d134188e25df..c6ed04a8943d 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -143,4 +143,17 @@ UI tests are used for macros to ensure that the output of a macro doesn’t chan These UI tests are sensible to any changes in the macro generated code or to switching the rust stable version. The tests are only run when the `RUN_UI_TESTS` environment variable is set. So, when the CI is for example complaining about failing UI tests and it is expected that they fail these tests need to be executed locally. -To simplify the updating of the UI test ouput there is the `.maintain/update-rust-stable +To simplify the updating of the UI test output there is a script +- `./scripts/update-ui-tests.sh` to update the tests for a current rust version locally +- `./scripts/update-ui-tests.sh 1.70` # to update the tests for a specific rust version locally + +Or if you have opened PR and you're member of `paritytech` - you can use command-bot to run the tests for you in CI: +- `bot update-ui` - will run the tests for the current rust version +- `bot update-ui latest --rust_version=1.70.0` - will run the tests for the specified rust version +- `bot update-ui latest -v CMD_IMAGE=paritytech/ci-unified:bullseye-1.70.0-2023-05-23 --rust_version=1.70.0` - will run the tests for the specified rust version and specified image + +## Command Bot + +If you're member of **paritytech** org - you can use command-bot to run various of common commands in CI: + +Start with comment in PR: `bot help` to see the list of available commands. diff --git a/polkadot/scripts/update-rust-stable.sh b/polkadot/scripts/update-rust-stable.sh deleted file mode 100755 index 6aae75d8cb2d..000000000000 --- a/polkadot/scripts/update-rust-stable.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# -# Script for updating the UI tests for a new rust stable version. -# -# It needs to be called like this: -# -# update-rust-stable.sh 1.61 -# -# This will run all UI tests with the rust stable 1.61. The script -# requires that rustup is installed. -set -e - -if [ "$#" -ne 1 ]; then - echo "Please specify the rust version to use. E.g. update-rust-stable.sh 1.61" - exit -fi - -RUST_VERSION=$1 - -if ! command -v rustup &> /dev/null -then - echo "rustup needs to be installed" - exit -fi - -rustup install $RUST_VERSION -rustup component add rust-src --toolchain $RUST_VERSION - -# Ensure we run the ui tests -export RUN_UI_TESTS=1 -# We don't need any wasm files for ui tests -export SKIP_WASM_BUILD=1 -# Let trybuild overwrite the .stderr files -export TRYBUILD=overwrite - -# Run all the relevant UI tests -# -# Any new UI tests in different crates need to be added here as well. -rustup run $RUST_VERSION cargo test -p orchestra ui diff --git a/polkadot/scripts/update-ui-tests.sh b/polkadot/scripts/update-ui-tests.sh new file mode 100644 index 000000000000..8f7a70b510dc --- /dev/null +++ b/polkadot/scripts/update-ui-tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e +# Run all the relevant UI tests +# Any new UI tests in different crates need to be added here as well. +$RUSTUP_RUN cargo test -p orchestra ui diff --git a/scripts/update-ui-tests.sh b/scripts/update-ui-tests.sh new file mode 100644 index 000000000000..7863988b857f --- /dev/null +++ b/scripts/update-ui-tests.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Script for updating the UI tests for a new rust stable version. +# Exit on error +set -e + +# by default current rust stable will be used +RUSTUP_RUN="" +# check if we have a parameter +# ./substrate/.maintain/update-rust-stable.sh 1.70 +if [ "$#" -e 1 ]; then + # This will run all UI tests with the rust stable 1.70. + # The script requires that rustup is installed. + RUST_VERSION=$1 + RUSTUP_RUN=rustup run $RUST_VERSION + + rustup install $RUST_VERSION + rustup component add rust-src --toolchain $RUST_VERSION + + + if ! command -v rustup &> /dev/null + then + echo "rustup needs to be installed" + exit + fi +fi + +sh "./substrate/.maintain/update-ui-tests.sh" +sh "./polkadot/scripts/update-ui-tests.sh" diff --git a/substrate/.maintain/update-rust-stable.sh b/substrate/.maintain/update-rust-stable.sh deleted file mode 100755 index b253bb410531..000000000000 --- a/substrate/.maintain/update-rust-stable.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Script for updating the UI tests for a new rust stable version. -# -# It needs to be called like this: -# -# update-rust-stable.sh 1.61 -# -# This will run all UI tests with the rust stable 1.61. The script -# requires that rustup is installed. -set -e - -if [ "$#" -ne 1 ]; then - echo "Please specify the rust version to use. E.g. update-rust-stable.sh 1.61" - exit -fi - -RUST_VERSION=$1 - -if ! command -v rustup &> /dev/null -then - echo "rustup needs to be installed" - exit -fi - -rustup install $RUST_VERSION -rustup component add rust-src --toolchain $RUST_VERSION - -# Ensure we run the ui tests -export RUN_UI_TESTS=1 -# We don't need any wasm files for ui tests -export SKIP_WASM_BUILD=1 -# Let trybuild overwrite the .stderr files -export TRYBUILD=overwrite - -# Run all the relevant UI tests -# -# Any new UI tests in different crates need to be added here as well. -rustup run $RUST_VERSION cargo test -p sp-runtime-interface ui -rustup run $RUST_VERSION cargo test -p sp-api-test ui -rustup run $RUST_VERSION cargo test -p frame-election-provider-solution-type ui -rustup run $RUST_VERSION cargo test -p frame-support-test ui diff --git a/substrate/.maintain/update-ui-tests.sh b/substrate/.maintain/update-ui-tests.sh new file mode 100644 index 000000000000..5c0b0f5f4556 --- /dev/null +++ b/substrate/.maintain/update-ui-tests.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -e +# Run all the relevant UI tests +# Any new UI tests in different crates need to be added here as well. +$RUSTUP_RUN cargo test -p sp-runtime-interface ui +$RUSTUP_RUN cargo test -p sp-api-test ui +$RUSTUP_RUN cargo test -p frame-election-provider-solution-type ui +$RUSTUP_RUN cargo test -p frame-support-test ui From d58fe325fcb784a8ef86daca183ecd803292ec7f Mon Sep 17 00:00:00 2001 From: Mak Date: Thu, 21 Sep 2023 18:17:59 +0100 Subject: [PATCH 2/9] Update CONTRIBUTING.md --- docs/CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index e6af8cad6d4e..1e05755a9b83 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -150,7 +150,8 @@ To simplify the updating of the UI test output there is a script Or if you have opened PR and you're member of `paritytech` - you can use command-bot to run the tests for you in CI: - `bot update-ui` - will run the tests for the current rust version - `bot update-ui latest --rust_version=1.70.0` - will run the tests for the specified rust version -- `bot update-ui latest -v CMD_IMAGE=paritytech/ci-unified:bullseye-1.70.0-2023-05-23 --rust_version=1.70.0` - will run the tests for the specified rust version and specified image +- `bot update-ui latest -v CMD_IMAGE=paritytech/ci-unified:bullseye-1.70.0-2023-05-23 --rust_version=1.70.0` - +will run the tests for the specified rust version and specified image ## Command Bot From a5fea034f52062901f01ac39e7cc61bf593b4472 Mon Sep 17 00:00:00 2001 From: Mak Date: Thu, 21 Sep 2023 18:55:36 +0100 Subject: [PATCH 3/9] combine into 1 scripts file --- polkadot/scripts/update-ui-tests.sh | 5 ----- scripts/update-ui-tests.sh | 11 +++++++++-- substrate/.maintain/update-ui-tests.sh | 8 -------- 3 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 polkadot/scripts/update-ui-tests.sh delete mode 100644 substrate/.maintain/update-ui-tests.sh diff --git a/polkadot/scripts/update-ui-tests.sh b/polkadot/scripts/update-ui-tests.sh deleted file mode 100644 index 8f7a70b510dc..000000000000 --- a/polkadot/scripts/update-ui-tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -e -# Run all the relevant UI tests -# Any new UI tests in different crates need to be added here as well. -$RUSTUP_RUN cargo test -p orchestra ui diff --git a/scripts/update-ui-tests.sh b/scripts/update-ui-tests.sh index 7863988b857f..690f7df12f49 100644 --- a/scripts/update-ui-tests.sh +++ b/scripts/update-ui-tests.sh @@ -24,5 +24,12 @@ if [ "$#" -e 1 ]; then fi fi -sh "./substrate/.maintain/update-ui-tests.sh" -sh "./polkadot/scripts/update-ui-tests.sh" + +# ./substrate +$RUSTUP_RUN cargo test -p sp-runtime-interface ui +$RUSTUP_RUN cargo test -p sp-api-test ui +$RUSTUP_RUN cargo test -p frame-election-provider-solution-type ui +$RUSTUP_RUN cargo test -p frame-support-test ui + +# ./polkadot +$RUSTUP_RUN cargo test -p orchestra ui diff --git a/substrate/.maintain/update-ui-tests.sh b/substrate/.maintain/update-ui-tests.sh deleted file mode 100644 index 5c0b0f5f4556..000000000000 --- a/substrate/.maintain/update-ui-tests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -e -# Run all the relevant UI tests -# Any new UI tests in different crates need to be added here as well. -$RUSTUP_RUN cargo test -p sp-runtime-interface ui -$RUSTUP_RUN cargo test -p sp-api-test ui -$RUSTUP_RUN cargo test -p frame-election-provider-solution-type ui -$RUSTUP_RUN cargo test -p frame-support-test ui From 6a5d5840e272a88d5ce9bf811c9543050edfdd2c Mon Sep 17 00:00:00 2001 From: Mak Date: Thu, 21 Sep 2023 18:56:00 +0100 Subject: [PATCH 4/9] Update update-ui-tests.sh --- scripts/update-ui-tests.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/update-ui-tests.sh b/scripts/update-ui-tests.sh index 690f7df12f49..99fb02421dff 100644 --- a/scripts/update-ui-tests.sh +++ b/scripts/update-ui-tests.sh @@ -24,6 +24,12 @@ if [ "$#" -e 1 ]; then fi fi +# Ensure we run the ui tests +export RUN_UI_TESTS=1 +# We don't need any wasm files for ui tests +export SKIP_WASM_BUILD=1 +# Let trybuild overwrite the .stderr files +export TRYBUILD=overwrite # ./substrate $RUSTUP_RUN cargo test -p sp-runtime-interface ui From fa813156ea92ca6850cc95e07dd347ffdf406e4b Mon Sep 17 00:00:00 2001 From: Mak Date: Thu, 21 Sep 2023 19:05:26 +0100 Subject: [PATCH 5/9] +x --- scripts/update-ui-tests.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/update-ui-tests.sh diff --git a/scripts/update-ui-tests.sh b/scripts/update-ui-tests.sh old mode 100644 new mode 100755 From 01d8b5e650436f9d00270287788ed05cc3f6f2ef Mon Sep 17 00:00:00 2001 From: Mak Date: Fri, 22 Sep 2023 16:31:48 +0100 Subject: [PATCH 6/9] remove orchestra --- scripts/update-ui-tests.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/update-ui-tests.sh b/scripts/update-ui-tests.sh index 99fb02421dff..099f62729c2e 100755 --- a/scripts/update-ui-tests.sh +++ b/scripts/update-ui-tests.sh @@ -36,6 +36,3 @@ $RUSTUP_RUN cargo test -p sp-runtime-interface ui $RUSTUP_RUN cargo test -p sp-api-test ui $RUSTUP_RUN cargo test -p frame-election-provider-solution-type ui $RUSTUP_RUN cargo test -p frame-support-test ui - -# ./polkadot -$RUSTUP_RUN cargo test -p orchestra ui From 00b3b93b3111cc7ebe20c6b392cc360c3bf91d33 Mon Sep 17 00:00:00 2001 From: Mak Date: Fri, 22 Sep 2023 18:29:38 +0100 Subject: [PATCH 7/9] Update update-ui-tests.sh --- scripts/update-ui-tests.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/update-ui-tests.sh b/scripts/update-ui-tests.sh index 099f62729c2e..e5bccd221f0f 100755 --- a/scripts/update-ui-tests.sh +++ b/scripts/update-ui-tests.sh @@ -7,12 +7,15 @@ set -e RUSTUP_RUN="" # check if we have a parameter # ./substrate/.maintain/update-rust-stable.sh 1.70 -if [ "$#" -e 1 ]; then +if [ ! -z "$1" ]; then + echo "RUST_VERSION: $1" # This will run all UI tests with the rust stable 1.70. # The script requires that rustup is installed. RUST_VERSION=$1 - RUSTUP_RUN=rustup run $RUST_VERSION + RUSTUP_RUN="rustup run $RUST_VERSION" + + echo "installing rustup $RUST_VERSION" rustup install $RUST_VERSION rustup component add rust-src --toolchain $RUST_VERSION From 95b2effcb089da6455af63203602726070c1edb4 Mon Sep 17 00:00:00 2001 From: Maksym H <1177472+mordamax@users.noreply.github.com> Date: Fri, 22 Sep 2023 18:30:15 +0100 Subject: [PATCH 8/9] Update scripts/update-ui-tests.sh Co-authored-by: Mira Ressel --- scripts/update-ui-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update-ui-tests.sh b/scripts/update-ui-tests.sh index e5bccd221f0f..58f03d8f10c1 100755 --- a/scripts/update-ui-tests.sh +++ b/scripts/update-ui-tests.sh @@ -6,7 +6,7 @@ set -e # by default current rust stable will be used RUSTUP_RUN="" # check if we have a parameter -# ./substrate/.maintain/update-rust-stable.sh 1.70 +# ./scripts/update-ui-tests.sh 1.70 if [ ! -z "$1" ]; then echo "RUST_VERSION: $1" # This will run all UI tests with the rust stable 1.70. From 3abddc7934bb734719e7dbe95973af3a44c42525 Mon Sep 17 00:00:00 2001 From: Maksym H <1177472+mordamax@users.noreply.github.com> Date: Fri, 22 Sep 2023 18:30:43 +0100 Subject: [PATCH 9/9] Update scripts/update-ui-tests.sh Co-authored-by: Mira Ressel --- scripts/update-ui-tests.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/update-ui-tests.sh b/scripts/update-ui-tests.sh index 58f03d8f10c1..785cc7bd3291 100755 --- a/scripts/update-ui-tests.sh +++ b/scripts/update-ui-tests.sh @@ -16,15 +16,14 @@ if [ ! -z "$1" ]; then echo "installing rustup $RUST_VERSION" - rustup install $RUST_VERSION - rustup component add rust-src --toolchain $RUST_VERSION - - if ! command -v rustup &> /dev/null then echo "rustup needs to be installed" exit fi + + rustup install $RUST_VERSION + rustup component add rust-src --toolchain $RUST_VERSION fi # Ensure we run the ui tests