Merge branch 'master' into pallet-assets-vesting

.github/workflows/bench-all-runtimes.yml

@@ -4,7 +4,11 @@ on:
   # schedule:
     # - cron: '0 1 * * 0' # weekly on Sunday night 01:00 UTC
   workflow_dispatch:
-  # pull_request:
+    inputs:
+      draft:
+        type: boolean
+        default: false
+        description: "Whether to create a draft PR"
 
 permissions: # allow the action to create a PR
   contents: write
@@ -22,19 +26,39 @@ jobs:
     timeout-minutes: 30
     outputs:
       runtime: ${{ steps.runtime.outputs.runtime }}
+      branch: ${{ steps.branch.outputs.branch }}
+      date: ${{ steps.branch.outputs.date }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     name: Extract runtimes from matrix
     steps:
       - uses: actions/checkout@v4
-      - id: runtime
+        with:
+          ref: master
+
+      - name: Extract runtimes
+        id: runtime
         run: |
           RUNTIMES=$(jq '[.[] | select(.package != null)]' .github/workflows/runtimes-matrix.json)
 
           RUNTIMES=$(echo $RUNTIMES | jq -c .)
           echo "runtime=$RUNTIMES"
           echo "runtime=$RUNTIMES" >> $GITHUB_OUTPUT
 
+      - name: Create branch
+        id: branch
+        run: |
+          DATE=$(date +'%Y-%m-%d-%s')
+          BRANCH="update-weights-weekly-$DATE"
+          # Fixes "detected dubious ownership" error in the ci
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+
+          git checkout -b $BRANCH
+          git push --set-upstream origin $BRANCH
+
+          echo "date=$DATE" >> $GITHUB_OUTPUT
+          echo "branch=$BRANCH" >> $GITHUB_OUTPUT
+
   run-frame-omni-bencher:
     needs: [preflight, runtime-matrix]
     runs-on: ${{ needs.preflight.outputs.RUNNER_WEIGHTS }}
@@ -58,11 +82,12 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: master
+          ref: ${{ needs.runtime-matrix.outputs.branch }} # checkout always from the initially created branch to avoid conflicts
 
       - name: script
         id: required
         run: |
+          git --version
           # Fixes "detected dubious ownership" error in the ci
           git config --global --add safe.directory $GITHUB_WORKSPACE
           git remote -v
@@ -94,21 +119,18 @@ jobs:
 
   apply-diff-commit:
     runs-on: ubuntu-latest
-    needs: [run-frame-omni-bencher]
+    needs: [runtime-matrix, run-frame-omni-bencher]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: master
+          ref: ${{ needs.runtime-matrix.outputs.branch }}
 
       - name: Download all artifacts
         uses: actions/download-artifact@v4
         with:
-          path: patches
-
-      - name: Install subweight
-        run: cargo install subweight
+          path: patches      
 
       # needs to be able to trigger CI
       - uses: actions/create-github-app-token@v1
@@ -120,28 +142,65 @@ jobs:
       - name: Apply diff and create PR
         env:
           GH_TOKEN: ${{ steps.generate_token.outputs.token }}
+          BRANCH: ${{ needs.runtime-matrix.outputs.branch }}
+          DATE: ${{ needs.runtime-matrix.outputs.date }}
         run: |
-          DATE=$(date +'%Y-%m-%d-%s')
-          BRANCH="update-weights-weekly-$DATE"
-          
+          git --version          
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+          git status
           
-          git switch -c "$BRANCH"
-          
+          # Apply all patches
           for file in patches/diff-*/diff-*.patch; do
             if [ -f "$file" ] && [ -s "$file" ]; then
               echo "Applying $file"
-              git apply "$file" --unidiff-zero --allow-empty || echo "Failed to apply $file"
+              # using --3way and --ours for conflicts resolution. Requires git 2.47+
+              git apply "$file" --unidiff-zero --allow-empty --3way --ours || echo "Failed to apply $file"
             else
               echo "Skipping empty or non-existent patch file: $file"
             fi
           done
+
           rm -rf patches
+
+          # Get release tags from 1 and 3 months ago
+          ONE_MONTH_AGO=$(date -d "1 month ago" +%Y-%m-%d)
+          THREE_MONTHS_AGO=$(date -d "3 months ago" +%Y-%m-%d)
+          
+          # Get tags with their dates
+          ONE_MONTH_INFO=$(git for-each-ref --sort=-creatordate --format '%(refname:short)|%(creatordate:iso-strict-local)' 'refs/tags/polkadot-v*' | awk -v date="$ONE_MONTH_AGO" -F'|' '$2 <= date {print $0; exit}')
+          THREE_MONTHS_INFO=$(git for-each-ref --sort=-creatordate --format '%(refname:short)|%(creatordate:iso-strict-local)' 'refs/tags/polkadot-v*' | awk -v date="$THREE_MONTHS_AGO" -F'|' '$2 <= date {print $0; exit}')
+          
+          # Split into tag and date
+          ONE_MONTH_TAG=$(echo "$ONE_MONTH_INFO" | cut -d'|' -f1)
+          ONE_MONTH_DATE=$(echo "$ONE_MONTH_INFO" | cut -d'|' -f2 | cut -d'T' -f1)
+          THREE_MONTHS_TAG=$(echo "$THREE_MONTHS_INFO" | cut -d'|' -f1)
+          THREE_MONTHS_DATE=$(echo "$THREE_MONTHS_INFO" | cut -d'|' -f2 | cut -d'T' -f1)
+
+          # Base URL for Subweight comparisons
+          BASE_URL="https://weights.tasty.limo/compare?repo=polkadot-sdk&threshold=5&path_pattern=.%2F**%2Fweights%2F**%2F*.rs%2C.%2F**%2Fweights.rs&method=asymptotic&ignore_errors=true&unit=time"
+          
+          # Generate comparison links
+          MASTER_LINK="${BASE_URL}&old=master&new=${BRANCH}"
+          ONE_MONTH_LINK="${BASE_URL}&old=${ONE_MONTH_TAG}&new=${BRANCH}"
+          THREE_MONTHS_LINK="${BASE_URL}&old=${THREE_MONTHS_TAG}&new=${BRANCH}"
+          
+          # Create PR body with all links in a temporary file
+          cat > /tmp/pr_body.md << EOF
+          Auto-update of all weights for ${DATE}.
+
+          Subweight results:
+          - [now vs master](${MASTER_LINK})
+          - [now vs ${ONE_MONTH_TAG} (${ONE_MONTH_DATE})](${ONE_MONTH_LINK})
+          - [now vs ${THREE_MONTHS_TAG} (${THREE_MONTHS_DATE})](${THREE_MONTHS_LINK})
+          EOF
           
           git add .
           git commit -m "Update all weights weekly for $DATE"
           git push --set-upstream origin "$BRANCH"
+
+          MAYBE_DRAFT=${{ inputs.draft && '--draft' || '' }}
           
           PR_TITLE="Auto-update of all weights for $DATE"
           gh pr create \
@@ -150,16 +209,6 @@ jobs:
             --base "master" \
             --reviewer paritytech/ci \
             --reviewer paritytech/release-engineering \
-            --draft \
+            $MAYBE_DRAFT \
             --label "R0-silent" \
-            --body "$PR_TITLE"
-
-          subweight compare commits \
-            --path-pattern "./**/weights/**/*.rs,./**/weights.rs" \
-            --method asymptotic \
-            --format markdown \
-            --no-color \
-            --change added changed \
-            --ignore-errors \
-            --threshold 2 \
-            origin/master $BRANCH
+            --body "$(cat /tmp/pr_body.md)"

.gitlab/pipeline/zombienet/polkadot.yml

@@ -464,3 +464,19 @@ zombienet-polkadot-functional-async-backing-6-seconds-rate:
     - unset NEXTEST_FAILURE_OUTPUT
     - unset NEXTEST_SUCCESS_OUTPUT
     - cargo nextest run --archive-file ./artifacts/polkadot-zombienet-tests.tar.zst --no-capture -- functional::async_backing_6_seconds_rate::async_backing_6_seconds_rate_test
+
+zombienet-polkadot-functional-duplicate-collations:
+  extends:
+    - .zombienet-polkadot-common
+  needs:
+    - job: build-polkadot-zombienet-tests
+      artifacts: true
+  before_script:
+    - !reference [ ".zombienet-polkadot-common", "before_script" ]
+    - export POLKADOT_IMAGE="${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+    - export X_INFRA_INSTANCE=spot # use spot by default
+  script:
+    # we want to use `--no-capture` in zombienet tests.
+    - unset NEXTEST_FAILURE_OUTPUT
+    - unset NEXTEST_SUCCESS_OUTPUT
+    - cargo nextest run --archive-file ./artifacts/polkadot-zombienet-tests.tar.zst --no-capture -- functional::duplicate_collations::duplicate_collations_test

Cargo.toml

@@ -741,8 +741,8 @@ cumulus-test-relay-sproof-builder = { path = "cumulus/test/relay-sproof-builder"
 cumulus-test-runtime = { path = "cumulus/test/runtime" }
 cumulus-test-service = { path = "cumulus/test/service" }
 curve25519-dalek = { version = "4.1.3" }
-derivative = { version = "2.2.0", default-features = false }
 derive-syn-parse = { version = "0.2.0" }
+derive-where = { version = "1.2.7" }
 derive_more = { version = "0.99.17", default-features = false }
 digest = { version = "0.10.3", default-features = false }
 directories = { version = "5.0.1" }

cumulus/pallets/weight-reclaim/Cargo.toml

@@ -27,7 +27,7 @@ frame-system = { workspace = true }
 
 # Other dependencies
 codec = { features = ["derive"], workspace = true }
-derivative = { features = ["use_core"], workspace = true }
+derive-where = { workspace = true }
 docify = { workspace = true }
 log = { workspace = true, default-features = true }
 scale-info = { features = ["derive"], workspace = true }

cumulus/pallets/weight-reclaim/src/lib.rs

@@ -29,7 +29,7 @@ extern crate alloc;
 use alloc::vec::Vec;
 use codec::{Decode, Encode};
 use cumulus_primitives_storage_weight_reclaim::get_proof_size;
-use derivative::Derivative;
+use derive_where::derive_where;
 use frame_support::{
 	dispatch::{DispatchInfo, PostDispatchInfo},
 	pallet_prelude::Weight,
@@ -83,13 +83,8 @@ pub mod pallet {
 /// calculates the unused weight using the post information and reclaim the unused weight.
 /// So this extension can be used as a drop-in replacement for `WeightReclaim` extension for
 /// parachains.
-#[derive(Encode, Decode, TypeInfo, Derivative)]
-#[derivative(
-	Clone(bound = "S: Clone"),
-	Eq(bound = "S: Eq"),
-	PartialEq(bound = "S: PartialEq"),
-	Default(bound = "S: Default")
-)]
+#[derive(Encode, Decode, TypeInfo)]
+#[derive_where(Clone, Eq, PartialEq, Default; S)]
 #[scale_info(skip_type_params(T))]
 pub struct StorageWeightReclaim<T, S>(pub S, core::marker::PhantomData<T>);
 

polkadot/node/test/service/src/chain_spec.rs

@@ -18,7 +18,8 @@
 
 use pallet_staking::Forcing;
 use polkadot_primitives::{
-	AccountId, AssignmentId, SchedulerParams, ValidatorId, MAX_CODE_SIZE, MAX_POV_SIZE,
+	node_features, AccountId, AssignmentId, NodeFeatures, SchedulerParams, ValidatorId,
+	MAX_CODE_SIZE, MAX_POV_SIZE,
 };
 use polkadot_service::chain_spec::Extensions;
 use polkadot_test_runtime::BABE_GENESIS_EPOCH_CONFIG;
@@ -110,6 +111,11 @@ fn polkadot_testnet_genesis(
 	const ENDOWMENT: u128 = 1_000_000 * DOTS;
 	const STASH: u128 = 100 * DOTS;
 
+	// Prepare node features with V2 receipts enabled.
+	let mut node_features = NodeFeatures::new();
+	node_features.resize(node_features::FeatureIndex::CandidateReceiptV2 as usize + 1, false);
+	node_features.set(node_features::FeatureIndex::CandidateReceiptV2 as u8 as usize, true);
+
 	serde_json::json!({
 		"balances": {
 			"balances": endowed_accounts.iter().map(|k| (k.clone(), ENDOWMENT)).collect::<Vec<_>>(),
@@ -158,6 +164,7 @@ fn polkadot_testnet_genesis(
 				no_show_slots: 10,
 				minimum_validation_upgrade_delay: 5,
 				max_downward_message_size: 1024,
+				node_features,
 				scheduler_params: SchedulerParams {
 					group_rotation_frequency: 20,
 					paras_availability_period: 4,

polkadot/parachain/test-parachains/undying/Cargo.toml

@@ -16,6 +16,7 @@ codec = { features = ["derive"], workspace = true }
 dlmalloc = { features = ["global"], workspace = true }
 log = { workspace = true }
 polkadot-parachain-primitives = { features = ["wasm-api"], workspace = true }
+polkadot-primitives = { workspace = true, default-features = false }
 tiny-keccak = { features = ["keccak"], workspace = true }
 
 # We need to make sure the global allocator is disabled until we have support of full substrate externalities
@@ -30,5 +31,6 @@ std = [
 	"codec/std",
 	"log/std",
 	"polkadot-parachain-primitives/std",
+	"polkadot-primitives/std",
 	"sp-io/std",
 ]

polkadot/parachain/test-parachains/undying/collator/Cargo.toml

@@ -22,13 +22,15 @@ futures-timer = { workspace = true }
 log = { workspace = true, default-features = true }
 
 polkadot-cli = { workspace = true, default-features = true }
+polkadot-erasure-coding = { workspace = true, default-features = true }
 polkadot-node-primitives = { workspace = true, default-features = true }
 polkadot-node-subsystem = { workspace = true, default-features = true }
 polkadot-primitives = { workspace = true, default-features = true }
 polkadot-service = { features = ["rococo-native"], workspace = true, default-features = true }
 test-parachain-undying = { workspace = true }
 
 sc-cli = { workspace = true, default-features = true }
+sc-client-api = { workspace = true, default-features = true }
 sc-service = { workspace = true, default-features = true }
 sp-core = { workspace = true, default-features = true }
 

polkadot/parachain/test-parachains/undying/collator/src/cli.rs

@@ -61,6 +61,15 @@ pub struct ExportGenesisWasmCommand {
 	pub output: Option<PathBuf>,
 }
 
+/// Enum representing different types of malicious behaviors for collators.
+#[derive(Debug, Parser, Clone, PartialEq, clap::ValueEnum)]
+pub enum MalusType {
+	/// No malicious behavior.
+	None,
+	/// Submit the same collations to all assigned cores.
+	DuplicateCollations,
+}
+
 #[allow(missing_docs)]
 #[derive(Debug, Parser)]
 #[group(skip)]
@@ -81,6 +90,10 @@ pub struct RunCmd {
 	/// we compute per block.
 	#[arg(long, default_value_t = 1)]
 	pub pvf_complexity: u32,
+
+	/// Specifies the malicious behavior of the collator.
+	#[arg(long, value_enum, default_value_t = MalusType::None)]
+	pub malus_type: MalusType,
 }
 
 #[allow(missing_docs)]