Merge remote-tracking branch 'origin/master' into wasmtime-sandbox

# Conflicts:
#	Cargo.lock
#	client/executor/wasmi/src/lib.rs
#	client/executor/wasmtime/Cargo.toml

.github/workflows/check-labels.yml

@@ -9,8 +9,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
       - name: Check labels
         run: bash ${{ github.workspace }}/.maintain/github/check_labels.sh
         env:
           GITHUB_PR: ${{ github.event.pull_request.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}

.github/workflows/monthly-tag.yml

@@ -0,0 +1,43 @@
+name: Monthly Snapshot Tag
+
+on:
+  schedule:
+    - cron: "0 1 1 * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Take Snapshot
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get the tags by date
+        id: tags
+        run: |
+          echo "::set-output name=new::$(date +'monthly-%Y-%m')"
+          echo "::set-output name=old::$(date -d'1 month ago' +'monthly-%Y-%m')"
+      - name: Checkout branch "master"
+        uses: actions/checkout@v2
+        with:
+          ref: 'master'
+          fetch-depth: 0
+      - name: Generate changelog
+        id: changelog
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "# Automatic snapshot pre-release ${{ steps.tags.outputs.new }}" > Changelog.md
+          echo "" >> Changelog.md
+          echo "## Changes since last snapshot (${{ steps.tags.outputs.old }})" >> Changelog.md
+          echo "" >> Changelog.md
+          ./.maintain/gitlab/generate_changelog.sh ${{ steps.tags.outputs.old }} >>  Changelog.md
+      - name: Release snapshot
+        id: release-snapshot
+        uses: actions/create-release@latest
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.tags.outputs.new }}
+          release_name: ${{ steps.tags.outputs.new }}
+          draft: false
+          prerelease: true
+          body_path: Changelog.md

.gitignore

@@ -24,3 +24,4 @@ rls*.log
 .cargo/
 .cargo-remote.toml
 *.bin
+*.iml

.gitlab-ci.yml

@@ -26,7 +26,6 @@ stages:
   - build
   - publish
   - deploy
-  - flaming-fir
 
 workflow:
   rules:
@@ -60,12 +59,15 @@ default:
     - kubernetes-parity-build
   interruptible:                   true
 
+.rust-info-script:                 &rust-info-script
+  - rustup show
+  - cargo --version
+  - sccache -s
+
 .docker-env:                       &docker-env
   image:                           "${CI_IMAGE}"
   before_script:
-    - rustup show
-    - cargo --version
-    - sccache -s
+    - *rust-info-script
   retry:
     max: 2
     when:
@@ -81,7 +83,6 @@ default:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
@@ -92,33 +93,49 @@ default:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
-.build-refs:                       &build-refs
+.test-refs-no-trigger-prs-only:   &test-refs-no-trigger-prs-only
   rules:
-    # .publish-refs with manual on PRs
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
     - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+    - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      when: manual
-      allow_failure: true
 
-.publish-refs:                     &publish-refs
+.build-refs:                       &build-refs
   rules:
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
+.nightly-pipeline:                 &nightly-pipeline
+  rules:
+    # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
+    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
+
+.merge-ref-into-master-script:     &merge-ref-into-master-script
+  - if [ $CI_COMMIT_REF_NAME != "master" ]; then
+      git fetch origin +master:master;
+      git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME;
+      git checkout master;
+      git config user.email "[email protected]";
+      git merge $CI_COMMIT_REF_NAME --verbose --no-edit;
+    fi
+
+.cargo-check-benches-script:       &cargo-check-benches-script
+  - mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
+  - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
+  - 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json
+    | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json'
+  - 'cargo run --release -p node-bench -- ::trie::read::small --json
+    | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
+  - sccache -s
+
 #### stage:                       .pre
 
 skip-if-draft:
@@ -199,17 +216,7 @@ test-prometheus-alerting-rules:
 cargo-deny:
   stage:                           test
   <<:                              *docker-env
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "pipeline"
-      when: never
-    - changes:
-      - "Cargo.lock"
-      - "**/Cargo.toml"
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+  <<:                              *nightly-pipeline
   script:
     - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml
   after_script:
@@ -227,12 +234,37 @@ cargo-deny:
 cargo-check-benches:
   stage:                           test
   <<:                              *docker-env
-  <<:                              *test-refs-no-trigger
+  <<:                              *test-refs
+  <<:                              *collect-artifacts
+  before_script:
+    # merges in the master branch on PRs
+    - *merge-ref-into-master-script
+    - *rust-info-script
   script:
-    - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
-    - cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small
-    - cargo run --release -p node-bench -- ::trie::read::small
-    - sccache -s
+    - *cargo-check-benches-script
+
+node-bench-regression-guard:
+  # it's not belong to `build` semantically, but dag jobs can't depend on each other
+  # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
+  # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
+  stage:                           build
+  <<:                              *docker-env
+  <<:                              *test-refs-no-trigger-prs-only
+  needs:
+    # this is a DAG
+    - job:                         cargo-check-benches
+      artifacts:                   true
+    # this does not like a DAG, just polls the artifact
+    - project:                     $CI_PROJECT_PATH
+      job:                         cargo-check-benches
+      ref:                         master
+      artifacts:                   true
+  variables:
+    CI_IMAGE:                      "paritytech/node-bench-regression-guard:latest"
+  before_script: [""]
+  script:
+    - 'node-bench-regression-guard --reference artifacts/benches/master-* 
+       --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
 
 cargo-check-subkey:
   stage:                           test
@@ -243,6 +275,14 @@ cargo-check-subkey:
     - SKIP_WASM_BUILD=1 time cargo check --release
     - sccache -s
 
+cargo-check-try-runtime:
+  stage:                           test
+  <<:                              *docker-env
+  <<:                              *test-refs
+  script:
+    - time cargo check --features try-runtime
+    - sccache -s
+
 test-deterministic-wasm:
   stage:                           test
   <<:                              *docker-env
@@ -277,7 +317,7 @@ test-linux-stable:                 &test-linux
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
     - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
-    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml
+    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml # does not reuse cache 1 min 44 sec
     - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 
@@ -288,7 +328,6 @@ unleash-check:
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   script:
     - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
@@ -341,7 +380,7 @@ check-web-wasm:
     # Note: we don't need to test crates imported in `bin/node/cli`
     - time cargo build --manifest-path=client/consensus/aura/Cargo.toml --target=wasm32-unknown-unknown --features getrandom
     # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way.
-    - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown -Z features=itarget
+    - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown
     # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
     - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features
     - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
@@ -412,7 +451,7 @@ test-browser-node:
     CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner"
     WASM_BINDGEN_TEST_TIMEOUT:                  120
   script:
-    - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing -Z features=itarget
+    - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing
 
 build-linux-substrate:             &build-binary
   stage:                           build
@@ -496,7 +535,7 @@ build-rust-doc:
 #### stage:                        publish
 
 .build-push-docker-image:          &build-push-docker-image
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   image:                           quay.io/buildah/stable
   variables:                       &docker-build-vars
@@ -524,24 +563,27 @@ build-rust-doc:
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
+  after_script:
     - buildah logout "$IMAGE_NAME"
+    # pass artifacts to the trigger-simnet job
+    - echo "IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/$PRODUCT/build.env
+    - echo "IMAGE_TAG=${VERSION}" >> ./artifacts/$PRODUCT/build.env
 
 publish-docker-substrate:
   stage:                           publish
   <<:                              *build-push-docker-image
+  <<:                              *build-refs
   needs:
     - job:                         build-linux-substrate
       artifacts:                   true
   variables:
     <<:                            *docker-build-vars
     PRODUCT:                       substrate
-  after_script:
-    - echo "VERSION=${VERSION}" >> build.env
   artifacts:
     reports:
       # this artifact is used in trigger-simnet job
       # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
-      dotenv: artifacts/substrate/build.env
+      dotenv: ./artifacts/substrate/build.env
 
 publish-docker-subkey:
   stage:                           publish
@@ -555,7 +597,7 @@ publish-docker-subkey:
 
 publish-s3-release:
   stage:                           publish
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   needs:
     - job:                         build-linux-substrate
@@ -584,7 +626,7 @@ publish-s3-doc:
       artifacts:                   true
     - job:                         build-linux-substrate
       artifacts:                   false
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   variables:
     GIT_STRATEGY:                  none
@@ -654,12 +696,21 @@ deploy-prometheus-alerting-rules:
 
 trigger-simnet:
   stage:                           deploy
+  image:                           paritytech/tools:latest
   rules:
-    # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
-    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "master"
   needs:
     - job:                         publish-docker-substrate
-  trigger:
-    project:                       parity/simnet
-    branch:                        master
-    strategy:                      depend
+  # `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$VERSION` here, 
+  # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
+  variables:
+    TRGR_PROJECT:                  ${CI_PROJECT_NAME}
+    TRGR_REF:                      ${CI_COMMIT_REF_NAME}
+    # simnet project ID
+    DWNSTRM_ID:                    332
+  script:
+    # API trigger for a simnet job
+    - ./scripts/gitlab/trigger_pipeline.sh

.maintain/common/lib.sh

@@ -82,7 +82,7 @@ has_label(){
 
 # Formats a message into a JSON string for posting to Matrix
 # message: 'any plaintext message'
-# formatted_message: '<strong>optional message formatted in <em>html</em></strong>' 
+# formatted_message: '<strong>optional message formatted in <em>html</em></strong>'
 # Usage: structure_message $content $formatted_content (optional)
 structure_message() {
   if [ -z "$2" ]; then
@@ -101,3 +101,17 @@ structure_message() {
 send_message() {
 curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
 }
+
+# Check for runtime changes between two commits. This is defined as any changes
+# to bin/node/src/runtime, frame/ and primitives/sr_* trees.
+has_runtime_changes() {
+  from=$1
+  to=$2
+  if git diff --name-only "${from}...${to}" \
+    | grep -q -e '^frame/' -e '^primitives/'
+  then
+    return 0
+  else
+    return 1
+  fi
+}