Merge branch 'master' into async-backing-docs

paritytech · Jun 28, 2024 · 83b4994 · 83b4994
2 parents 6b53c0e + 18a6a56
commit 83b4994
Show file tree

Hide file tree

Showing 810 changed files with 19,187 additions and 12,847 deletions.
diff --git a/.config/taplo.toml b/.config/taplo.toml
@@ -33,3 +33,10 @@ keys = ["build"]
 
 [rule.formatting]
 reorder_arrays = false
+
+[[rule]]
+include = ["Cargo.toml"]
+keys = ["workspace.dependencies"]
+
+[rule.formatting]
+reorder_keys = true
diff --git a/.forklift/config-gitlab.toml b/.forklift/config-gitlab.toml
@@ -0,0 +1,33 @@
+[compression]
+type = "zstd"
+
+[compression.zstd]
+compressionLevel = 3
+
+[general]
+jobNameVariable = "CI_JOB_NAME"
+jobsBlackList = []
+logLevel = "warn"
+threadsCount = 6
+
+[cache]
+extraEnv = ["RUNTIME_METADATA_HASH"]
+
+[metrics]
+enabled = true
+pushEndpoint = "placeholder"
+
+[metrics.extraLabels]
+environment = "production"
+job_name = "$CI_JOB_NAME"
+project_name = "$CI_PROJECT_PATH"
+
+[storage]
+type = "s3"
+
+[storage.s3]
+accessKeyId = "placeholder"
+bucketName = "placeholder"
+concurrency = 10
+endpointUrl = "placeholder"
+secretAccessKey = "placeholder"
diff --git a/.forklift/config.toml b/.forklift/config.toml
@@ -23,11 +23,7 @@ job_name = "$CI_JOB_NAME"
 project_name = "$CI_PROJECT_PATH"
 
 [storage]
-type = "s3"
+type = "gcs"
 
-[storage.s3]
-accessKeyId = "placeholder"
-bucketName = "placeholder"
-concurrency = 10
-endpointUrl = "placeholder"
-secretAccessKey = "placeholder"
+[storage.gcs]
+bucketName = "parity-ci-forklift"
diff --git a/.github/review-bot.yml b/.github/review-bot.yml
@@ -9,6 +9,7 @@ rules:
         - ^\.gitlab/.*
         - ^\.config/nextest.toml
         - ^\.cargo/.*
+        - ^\.forklift/.*
       exclude:
         - ^\.gitlab/pipeline/zombienet.*
     type: "or"
@@ -33,6 +34,7 @@ rules:
         - ^docker/.*
         - ^\.github/.*
         - ^\.gitlab/.*
+        - ^\.forklift/.*
         - ^\.config/nextest.toml
         - ^\.cargo/.*
     minApprovals: 2

diff --git a/.github/scripts/check-workspace.py b/.github/scripts/check-workspace.py
@@ -135,8 +135,12 @@ def check_deps(deps):
 				if dep_name in all_crates:
 					links.append((name, dep_name))
 
-					if not 'path' in deps[dep]:
-						broken.append((name, dep_name, "crate must be linked via `path`"))
+					if name == 'polkadot-sdk':
+						if not 'path' in deps[dep]:
+							broken.append((name, dep_name, "crate must use path"))
+							return
+					elif not 'workspace' in deps[dep] or not deps[dep]['workspace']:
+						broken.append((name, dep_name, "crate must use workspace inheritance"))
 						return
 
 		def check_crate(deps):
@@ -154,8 +158,6 @@ def check_crate(deps):
 
 		check_crate(manifest)
 
-
-
 	links.sort()
 	broken.sort()
 

diff --git a/.github/scripts/deny-git-deps.py b/.github/scripts/deny-git-deps.py
@@ -19,6 +19,7 @@
 
 root = sys.argv[1] if len(sys.argv) > 1 else os.getcwd()
 workspace = Workspace.from_path(root)
+errors = []
 
 def check_dep(dep, used_by):
 	if dep.location != DependencyLocation.GIT:
@@ -27,14 +28,23 @@ def check_dep(dep, used_by):
 	if used_by in KNOWN_BAD_GIT_DEPS.get(dep.name, []):
 		print(f'🤨 Ignoring git dependency {dep.name} in {used_by}')
 	else:
-		print(f'🚫 Found git dependency {dep.name} in {used_by}')
-		sys.exit(1)	
+		errors.append(f'🚫 Found git dependency {dep.name} in {used_by}')
 
 # Check the workspace dependencies that can be inherited:
 for dep in workspace.dependencies:
 	check_dep(dep, "workspace")
 
+	if workspace.crates.find_by_name(dep.name):
+		if dep.location != DependencyLocation.PATH:
+			errors.append(f'🚫 Workspace must use path to link local dependency {dep.name}')
+
 # And the dependencies of each crate:
 for crate in workspace.crates:
 	for dep in crate.dependencies:
 		check_dep(dep, crate.name)
+
+if errors:
+	print('❌ Found errors:')
+	for error in errors:
+		print(error)
+	sys.exit(1)
diff --git a/.github/workflows/check-runtime-migration.yml b/.github/workflows/check-runtime-migration.yml
@@ -11,13 +11,6 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-env:
-  FORKLIFT_storage_s3_bucketName: ${{ secrets.FORKLIFT_storage_s3_bucketName }}
-  FORKLIFT_storage_s3_accessKeyId: ${{ secrets.FORKLIFT_storage_s3_accessKeyId }}
-  FORKLIFT_storage_s3_secretAccessKey: ${{ secrets.FORKLIFT_storage_s3_secretAccessKey }}
-  FORKLIFT_storage_s3_endpointUrl: ${{ secrets.FORKLIFT_storage_s3_endpointUrl }}
-  FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }}
-
 jobs:
   set-image:
     # GitHub Actions allows using 'env' in a container context.

diff --git a/.github/workflows/check-semver.yml b/.github/workflows/check-semver.yml
@@ -5,6 +5,8 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
     paths:
       - prdoc/*.prdoc
+env:
+  TOOLCHAIN: nightly-2024-03-01
 
 jobs:
   check-semver:
@@ -19,14 +21,14 @@ jobs:
         with:
           cache-on-failure: true
 
+      - name: install parity-publish
+        run: cargo install [email protected]
+
       - name: Rust compilation prerequisites
         run: |
-          rustup default nightly-2024-03-01
-          rustup target add wasm32-unknown-unknown --toolchain nightly-2024-03-01
-          rustup component add rust-src --toolchain nightly-2024-03-01
-
-      - name: install parity-publish
-        run: cargo install [email protected]
+          rustup default $TOOLCHAIN
+          rustup target add wasm32-unknown-unknown --toolchain $TOOLCHAIN
+          rustup component add rust-src --toolchain $TOOLCHAIN
 
       - name: extra git setup
         run: |
@@ -39,7 +41,7 @@ jobs:
           export CARGO_TARGET_DIR=target
           export RUSTFLAGS='-A warnings -A missing_docs'
           export SKIP_WASM_BUILD=1
-          if ! parity-publish --color always prdoc --since old --validate prdoc/pr_$PR.prdoc --toolchain nightly-2024-03-01 -v; then
+          if ! parity-publish --color always prdoc --since old --validate prdoc/pr_$PR.prdoc -v --toolchain $TOOLCHAIN; then
             cat <<EOF
           👋 Hello developer! The SemVer information that you declared in the prdoc file did not match what the CI detected.
 

diff --git a/.github/workflows/checks-quick.yml b/.github/workflows/checks-quick.yml
@@ -55,8 +55,14 @@ jobs:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: fetch deps
+        run: |
+          # Pull all dependencies eagerly:
+          time cargo metadata --format-version=1 --locked > /dev/null
       - name: run zepter
-        run: zepter run check
+        run: |
+          zepter --version
+          time zepter run check
   test-rust-features:
     runs-on: ubuntu-latest
     timeout-minutes: 10

diff --git a/.github/workflows/publish-check-crates.yml b/.github/workflows/publish-check-crates.yml
@@ -20,7 +20,7 @@ jobs:
           cache-on-failure: true
 
       - name: install parity-publish
-        run: cargo install parity-publish@0.5.1
+        run: cargo install parity-publish@0.6.0
 
       - name: parity-publish check
         run: parity-publish --color always check --allow-unpublished
diff --git a/.github/workflows/publish-claim-crates.yml b/.github/workflows/publish-claim-crates.yml
@@ -18,7 +18,7 @@ jobs:
           cache-on-failure: true
 
       - name: install parity-publish
-        run: cargo install parity-publish@0.5.1
+        run: cargo install parity-publish@0.6.0
 
       - name: parity-publish claim
         env:

diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml
@@ -12,15 +12,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-env:
-  FORKLIFT_storage_s3_bucketName: ${{ secrets.FORKLIFT_storage_s3_bucketName }}
-  FORKLIFT_storage_s3_accessKeyId: ${{ secrets.FORKLIFT_storage_s3_accessKeyId }}
-  FORKLIFT_storage_s3_secretAccessKey: ${{ secrets.FORKLIFT_storage_s3_secretAccessKey }}
-  FORKLIFT_storage_s3_endpointUrl: ${{ secrets.FORKLIFT_storage_s3_endpointUrl }}
-  FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }}
-
 jobs:
-
   changes:
     permissions:
       pull-requests: read
@@ -31,7 +23,7 @@ jobs:
     # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
     # This workaround sets the container image for each job using 'set-image' job output.
     needs: changes
-    if: ${{ needs.changes.outputs.rust }}      
+    if: ${{ needs.changes.outputs.rust }}
     runs-on: ubuntu-latest
     outputs:
       IMAGE: ${{ steps.set_image.outputs.IMAGE }}
@@ -40,10 +32,10 @@ jobs:
         uses: actions/checkout@v4
       - id: set_image
         run: cat .github/env >> $GITHUB_OUTPUT
-  
+
   test-linux-stable-int:
     needs: [set-image, changes]
-    if: ${{ needs.changes.outputs.rust }}  
+    if: ${{ needs.changes.outputs.rust }}
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
     container:
@@ -60,11 +52,11 @@ jobs:
         uses: actions/checkout@v4
       - name: script
         run: WASM_BUILD_NO_COLOR=1 time forklift cargo test -p staging-node-cli --release --locked -- --ignored
-  
+
   # https://github.com/paritytech/ci_cd/issues/864
   test-linux-stable-runtime-benchmarks:
     needs: [set-image, changes]
-    if: ${{ needs.changes.outputs.rust }}    
+    if: ${{ needs.changes.outputs.rust }}
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
     container:

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -11,15 +11,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-env:
-  FORKLIFT_storage_s3_bucketName: ${{ secrets.FORKLIFT_storage_s3_bucketName }}
-  FORKLIFT_storage_s3_accessKeyId: ${{ secrets.FORKLIFT_storage_s3_accessKeyId }}
-  FORKLIFT_storage_s3_secretAccessKey: ${{ secrets.FORKLIFT_storage_s3_secretAccessKey }}
-  FORKLIFT_storage_s3_endpointUrl: ${{ secrets.FORKLIFT_storage_s3_endpointUrl }}
-  FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }}
-
 jobs:
-
   changes:
     permissions:
       pull-requests: read
@@ -40,7 +32,7 @@ jobs:
 
   quick-benchmarks:
     needs: [set-image, changes]
-    if: ${{ needs.changes.outputs.rust }}    
+    if: ${{ needs.changes.outputs.rust }}
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
     container:
@@ -55,11 +47,11 @@ jobs:
         uses: actions/checkout@v4
       - name: script
         run: time forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
-  
+
   # cf https://github.com/paritytech/polkadot-sdk/issues/1652
   test-syscalls:
     needs: [set-image, changes]
-    if: ${{ needs.changes.outputs.rust }}    
+    if: ${{ needs.changes.outputs.rust }}
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
     container:
@@ -81,10 +73,10 @@ jobs:
     # - if [[ "$CI_JOB_STATUS" == "failed" ]]; then
     #   printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
     #   fi
-  
+
   cargo-check-all-benches:
     needs: [set-image, changes]
-    if: ${{ needs.changes.outputs.rust }}    
+    if: ${{ needs.changes.outputs.rust }}
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
     container:

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -120,7 +120,8 @@ default:
 .forklift-cache:
   before_script:
     - mkdir ~/.forklift
-    - cp .forklift/config.toml ~/.forklift/config.toml
+    - cp .forklift/config-gitlab.toml ~/.forklift/config.toml
+    - cat .forklift/config-gitlab.toml > .forklift/config.toml
     - >
       if [ "$FORKLIFT_BYPASS" != "true" ]; then
         echo "FORKLIFT_BYPASS not set"; 

diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml
@@ -162,6 +162,9 @@ zombienet-polkadot-elastic-scaling-0001-basic-3cores-6s-blocks:
     - .zombienet-polkadot-common
   variables:
     FORCED_INFRA_INSTANCE: "spot-iops"
+  before_script:
+    - !reference [.zombienet-polkadot-common, before_script]
+    - cp --remove-destination ${LOCAL_DIR}/assign-core.js ${LOCAL_DIR}/elastic_scaling
   script:
     - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
       --local-dir="${LOCAL_DIR}/elastic_scaling"
@@ -170,6 +173,9 @@ zombienet-polkadot-elastic-scaling-0001-basic-3cores-6s-blocks:
 zombienet-polkadot-elastic-scaling-0002-elastic-scaling-doesnt-break-parachains:
   extends:
     - .zombienet-polkadot-common
+  before_script:
+    - !reference [.zombienet-polkadot-common, before_script]
+    - cp --remove-destination ${LOCAL_DIR}/assign-core.js ${LOCAL_DIR}/elastic_scaling
   script:
     - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
       --local-dir="${LOCAL_DIR}/elastic_scaling"
@@ -199,6 +205,17 @@ zombienet-polkadot-functional-0014-chunk-fetching-network-compatibility:
       --local-dir="${LOCAL_DIR}/functional"
       --test="0014-chunk-fetching-network-compatibility.zndsl"
 
+zombienet-polkadot-functional-0015-coretime-shared-core:
+  extends:
+    - .zombienet-polkadot-common
+  before_script:
+    - !reference [.zombienet-polkadot-common, before_script]
+    - cp --remove-destination ${LOCAL_DIR}/assign-core.js ${LOCAL_DIR}/functional
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}/functional"
+      --test="0015-coretime-shared-core.zndsl"
+
 zombienet-polkadot-smoke-0001-parachains-smoke-test:
   extends:
     - .zombienet-polkadot-common