P 970 pallet state trie migration (#3242)

* P-970: add pallet-state-trie-migration and update state_version to 1

* fix: typo

* fix: update the state_trie_migration pallet index to 250+

* fix: update the state_trie_migration pallet index to 251

---------

Co-authored-by: higherordertech <higherordertech>

parachain/Cargo.toml

@@ -226,6 +226,7 @@ pallet-vesting = { git = "https://github.com/paritytech/polkadot-sdk", branch =
 pallet-utility = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false }
 pallet-whitelist = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false }
 pallet-session = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false }
+pallet-state-trie-migration = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false }
 pallet-sudo = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false }
 pallet-tips = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false }
 pallet-timestamp = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false }

parachain/runtime/litentry/Cargo.toml

@@ -49,6 +49,7 @@ pallet-preimage = { workspace = true }
 pallet-proxy = { workspace = true }
 pallet-scheduler = { workspace = true }
 pallet-session = { workspace = true }
+pallet-state-trie-migration = { workspace = true }
 pallet-timestamp = { workspace = true }
 pallet-transaction-payment = { workspace = true }
 pallet-transaction-payment-rpc-runtime-api = { workspace = true }
@@ -174,6 +175,7 @@ runtime-benchmarks = [
     "pallet-proxy/runtime-benchmarks",
     "pallet-scheduler/runtime-benchmarks",
     "pallet-score-staking/runtime-benchmarks",
+    "pallet-state-trie-migration/runtime-benchmarks",
     "pallet-teebag/runtime-benchmarks",
     "pallet-timestamp/runtime-benchmarks",
     "pallet-treasury/runtime-benchmarks",
@@ -262,6 +264,7 @@ std = [
     "pallet-scheduler/std",
     "pallet-score-staking/std",
     "pallet-session/std",
+    "pallet-state-trie-migration/std",
     "pallet-teebag/std",
     "pallet-timestamp/std",
     "pallet-transaction-payment-rpc-runtime-api/std",
@@ -341,6 +344,7 @@ try-runtime = [
     "pallet-scheduler/try-runtime",
     "pallet-score-staking/try-runtime",
     "pallet-session/try-runtime",
+    "pallet-state-trie-migration/try-runtime",
     "pallet-teebag/try-runtime",
     "pallet-timestamp/try-runtime",
     "pallet-transaction-payment/try-runtime",

parachain/runtime/litentry/src/lib.rs

@@ -234,7 +234,15 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	impl_version: 0,
 	apis: RUNTIME_API_VERSIONS,
 	transaction_version: 1,
-	state_version: 0,
+	// https://hackmd.io/JagpUd8tTjuKf9HQtpvHIQ
+	// The trie is an abstraction that sits between the Runtime (and its Overlays) and the actual database, providing an important abstraction to the blockchain, namely storage proofs and state roots.
+	// The trie format has changed since this pull request(#9732) in substrate. The main new difference is, that nodes that contain values larger than 256 bits will not storage the value itself, but rather store the hash of that value. The value itself, is consequently stored in the node that lives in the path traversed by this new hash.
+	// The main benefit of this optimization is better PoV (proof of validity) size for parachains, since large values are moved out of the common trie paths.
+	// The new trie has been included in Polkadot client since release v0.9.16. Although, new new trie format is not yet enabled. This is only done once state_version in RuntimeVersion is set to 1. Once set to 1, the trie works in a hybrid format, meaning that no migration is needed. Instead, migration is done lazily on the fly. Any storage key that's written to will be migrated, if needed. This means that a part of all chain's state is will migrated to the new format pretty soon after setting state_version to 1.
+	// Nonetheless, it might take a long time for all chain's entire state to be migrated to the new format. The sooner this happens, the better, since the lazy migration is a small overhead. Moreover, this hybrid/lazy state mode does not support warp-sync and state import/export.
+	// To do this faster, we have developed pallet-state-trie-migration. This pallet is a configurable background task that starts reading and writing all keys in the storage based on some given schedule, until they are all read, ergo migrated. This pallet can be deployed to a runtime to make sure all keys are read/written once, to ensure that all trie nodes are migrated to the new format.
+	// All substrate-based chains are advised to switch their state_version to 1, and use this pallet to migrate to the new trie format as soon as they can. Switching the state_version will enable the hybrid, lazy migration mode, and this pallet will speed up the migration process.
+	state_version: 1,
 };
 
 /// The version information used to identify this runtime when compiled natively.
@@ -1355,6 +1363,8 @@ construct_runtime! {
 		Ethereum: pallet_ethereum = 121,
 
 		// TMP
+		// State Trie Migration
+		StateTrieMigration: pallet_state_trie_migration = 251,
 		AccountFix: pallet_account_fix = 254,
 	}
 }
@@ -2153,3 +2163,25 @@ cumulus_pallet_parachain_system::register_validate_block! {
 	Runtime = Runtime,
 	BlockExecutor = cumulus_pallet_aura_ext::BlockExecutor::<Runtime, Executive>,
 }
+
+parameter_types! {
+	// The deposit configuration for the signed migration. Specially if you want to allow any signed account to do the migration (see `SignedFilter`, these deposits should be high)
+	pub MigrationSignedDepositPerItem: Balance = 1 * CENTS;
+	pub MigrationSignedDepositBase: Balance = 20 * DOLLARS;
+	pub const MigrationMaxKeyLen: u32 = 512;
+}
+
+impl pallet_state_trie_migration::Config for Runtime {
+	// An origin that can control the whole pallet: should be Root, or a part of your council.
+	type ControlOrigin = EnsureRootOrTwoThirdsTechnicalCommittee;
+	// specific account for the migration, can trigger the signed migrations.
+	type SignedFilter = frame_support::traits::NeverEnsureOrigin<AccountId>;
+	type RuntimeEvent = RuntimeEvent;
+	type Currency = Balances;
+	type RuntimeHoldReason = RuntimeHoldReason;
+	type MaxKeyLen = MigrationMaxKeyLen;
+	type SignedDepositPerItem = MigrationSignedDepositPerItem;
+	type SignedDepositBase = MigrationSignedDepositBase;
+	// Replace this with weight based on your runtime.
+	type WeightInfo = pallet_state_trie_migration::weights::SubstrateWeight<Runtime>;
+}

parachain/runtime/paseo/Cargo.toml

@@ -47,6 +47,7 @@ pallet-proxy = { workspace = true }
 pallet-referenda = { workspace = true }
 pallet-scheduler = { workspace = true }
 pallet-session = { workspace = true }
+pallet-state-trie-migration = { workspace = true }
 pallet-sudo = { workspace = true }
 pallet-timestamp = { workspace = true }
 pallet-tips = { workspace = true }
@@ -193,6 +194,7 @@ runtime-benchmarks = [
     "pallet-referenda/runtime-benchmarks",
     "pallet-scheduler/runtime-benchmarks",
     "pallet-score-staking/runtime-benchmarks",
+    "pallet-state-trie-migration/runtime-benchmarks",
     "pallet-sudo/runtime-benchmarks",
     "pallet-teebag/runtime-benchmarks",
     "pallet-timestamp/runtime-benchmarks",
@@ -293,6 +295,7 @@ std = [
     "pallet-scheduler/std",
     "pallet-score-staking/std",
     "pallet-session/std",
+    "pallet-state-trie-migration/std",
     "pallet-sudo/std",
     "pallet-teebag/std",
     "pallet-timestamp/std",
@@ -388,6 +391,7 @@ try-runtime = [
     "pallet-scheduler/try-runtime",
     "pallet-score-staking/try-runtime",
     "pallet-session/try-runtime",
+    "pallet-state-trie-migration/try-runtime",
     "pallet-sudo/try-runtime",
     "pallet-teebag/try-runtime",
     "pallet-timestamp/try-runtime",

parachain/runtime/paseo/src/lib.rs

@@ -237,7 +237,15 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	impl_version: 0,
 	apis: RUNTIME_API_VERSIONS,
 	transaction_version: 1,
-	state_version: 0,
+	// https://hackmd.io/JagpUd8tTjuKf9HQtpvHIQ
+	// The trie is an abstraction that sits between the Runtime (and its Overlays) and the actual database, providing an important abstraction to the blockchain, namely storage proofs and state roots.
+	// The trie format has changed since this pull request(#9732) in substrate. The main new difference is, that nodes that contain values larger than 256 bits will not storage the value itself, but rather store the hash of that value. The value itself, is consequently stored in the node that lives in the path traversed by this new hash.
+	// The main benefit of this optimization is better PoV (proof of validity) size for parachains, since large values are moved out of the common trie paths.
+	// The new trie has been included in Polkadot client since release v0.9.16. Although, new new trie format is not yet enabled. This is only done once state_version in RuntimeVersion is set to 1. Once set to 1, the trie works in a hybrid format, meaning that no migration is needed. Instead, migration is done lazily on the fly. Any storage key that's written to will be migrated, if needed. This means that a part of all chain's state is will migrated to the new format pretty soon after setting state_version to 1.
+	// Nonetheless, it might take a long time for all chain's entire state to be migrated to the new format. The sooner this happens, the better, since the lazy migration is a small overhead. Moreover, this hybrid/lazy state mode does not support warp-sync and state import/export.
+	// To do this faster, we have developed pallet-state-trie-migration. This pallet is a configurable background task that starts reading and writing all keys in the storage based on some given schedule, until they are all read, ergo migrated. This pallet can be deployed to a runtime to make sure all keys are read/written once, to ensure that all trie nodes are migrated to the new format.
+	// All substrate-based chains are advised to switch their state_version to 1, and use this pallet to migrate to the new trie format as soon as they can. Switching the state_version will enable the hybrid, lazy migration mode, and this pallet will speed up the migration process.
+	state_version: 1,
 };
 
 /// A timestamp: milliseconds since the unix epoch.
@@ -1505,6 +1513,8 @@ construct_runtime! {
 		Whitelist: pallet_whitelist::{Pallet, Call, Storage, Event<T>} = 173,
 
 		// TMP
+		// State Trie Migration
+		StateTrieMigration: pallet_state_trie_migration = 251,
 		AccountFix: pallet_account_fix = 254,
 		Sudo: pallet_sudo = 255,
 	}
@@ -2324,3 +2334,25 @@ cumulus_pallet_parachain_system::register_validate_block! {
 	Runtime = Runtime,
 	BlockExecutor = cumulus_pallet_aura_ext::BlockExecutor::<Runtime, Executive>,
 }
+
+parameter_types! {
+	// The deposit configuration for the signed migration. Specially if you want to allow any signed account to do the migration (see `SignedFilter`, these deposits should be high)
+	pub MigrationSignedDepositPerItem: Balance = 1 * CENTS;
+	pub MigrationSignedDepositBase: Balance = 20 * DOLLARS;
+	pub const MigrationMaxKeyLen: u32 = 512;
+}
+
+impl pallet_state_trie_migration::Config for Runtime {
+	// An origin that can control the whole pallet: should be Root, or a part of your council.
+	type ControlOrigin = EnsureRootOrTwoThirdsTechnicalCommittee;
+	// specific account for the migration, can trigger the signed migrations.
+	type SignedFilter = frame_support::traits::NeverEnsureOrigin<AccountId>;
+	type RuntimeEvent = RuntimeEvent;
+	type Currency = Balances;
+	type RuntimeHoldReason = RuntimeHoldReason;
+	type MaxKeyLen = MigrationMaxKeyLen;
+	type SignedDepositPerItem = MigrationSignedDepositPerItem;
+	type SignedDepositBase = MigrationSignedDepositBase;
+	// Replace this with weight based on your runtime.
+	type WeightInfo = pallet_state_trie_migration::weights::SubstrateWeight<Runtime>;
+}