Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

frame_support::storage: Add StorageStreamIter #12721

Merged
merged 20 commits into from
Dec 17, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
More
  • Loading branch information
bkchr committed Nov 11, 2022
commit 59a74b18df148b9b4b460ba1d0322948f96d04d0
94 changes: 57 additions & 37 deletions frame/support/src/storage/stream_iter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,40 +16,71 @@
// limitations under the License.

use crate::{BoundedBTreeMap, BoundedBTreeSet, BoundedVec, WeakBoundedVec};
use codec::Decode;
use sp_std::vec::Vec;

pub trait ScaleContainerType: private::Sealed {
type Stored: codec::Decode;
pub trait StreamIter: private::Sealed {
type Iterator: sp_std::iter::Iterator;

fn stream_iter(key: Vec<u8>) -> Self::Iterator;
}

impl<T: codec::Decode> ScaleContainerType for Vec<T> {
type Stored = T;
impl<T: codec::Decode> StreamIter for Vec<T> {
type Iterator = ScaleContainerStreamIter<T>;

fn stream_iter(key: Vec<u8>) -> Self::Iterator {
ScaleContainerStreamIter::new(key).unwrap()
}
}

impl<T: codec::Decode> ScaleContainerType for sp_std::collections::btree_set::BTreeSet<T> {
type Stored = T;
impl<T: codec::Decode> StreamIter for sp_std::collections::btree_set::BTreeSet<T> {
type Iterator = ScaleContainerStreamIter<T>;

fn stream_iter(key: Vec<u8>) -> Self::Iterator {
ScaleContainerStreamIter::new(key).unwrap()
}
}

impl<K: codec::Decode, V: codec::Decode> ScaleContainerType
impl<K: codec::Decode, V: codec::Decode> StreamIter
for sp_std::collections::btree_map::BTreeMap<K, V>
{
type Stored = (K, V);
type Iterator = ScaleContainerStreamIter<(K, V)>;

fn stream_iter(key: Vec<u8>) -> Self::Iterator {
ScaleContainerStreamIter::new(key).unwrap()
}
}

impl<T: codec::Decode, S> ScaleContainerType for BoundedVec<T, S> {
type Stored = T;
impl<T: codec::Decode, S> StreamIter for BoundedVec<T, S> {
type Iterator = ScaleContainerStreamIter<T>;

fn stream_iter(key: Vec<u8>) -> Self::Iterator {
ScaleContainerStreamIter::new(key).unwrap()
}
}

impl<T: codec::Decode, S> ScaleContainerType for WeakBoundedVec<T, S> {
type Stored = T;
impl<T: codec::Decode, S> StreamIter for WeakBoundedVec<T, S> {
type Iterator = ScaleContainerStreamIter<T>;

fn stream_iter(key: Vec<u8>) -> Self::Iterator {
ScaleContainerStreamIter::new(key).unwrap()
}
}

impl<K: codec::Decode, V: codec::Decode, S> ScaleContainerType for BoundedBTreeMap<K, V, S> {
type Stored = (K, V);
impl<K: codec::Decode, V: codec::Decode, S> StreamIter for BoundedBTreeMap<K, V, S> {
type Iterator = ScaleContainerStreamIter<(K, V)>;

fn stream_iter(key: Vec<u8>) -> Self::Iterator {
ScaleContainerStreamIter::new(key).unwrap()
}
}

impl<T: codec::Decode, S> ScaleContainerType for BoundedBTreeSet<T, S> {
type Stored = T;
impl<T: codec::Decode, S> StreamIter for BoundedBTreeSet<T, S> {
type Iterator = ScaleContainerStreamIter<T>;

fn stream_iter(key: Vec<u8>) -> Self::Iterator {
ScaleContainerStreamIter::new(key).unwrap()
}
}

mod private {
Expand All @@ -67,19 +98,15 @@ mod private {
impl<T: codec::Decode, S> Sealed for BoundedBTreeSet<T, S> {}
}

pub trait StreamIter {
type Iterator: sp_std::iter::Iterator;

fn stream_iter() -> Self::Iterator;
pub trait StreamIterExt<T: StreamIter> {
fn stream_iter() -> T::Iterator;
}

impl<T: ScaleContainerType, StorageValue: super::generator::StorageValue<T>> StreamIter
impl<T: StreamIter + codec::FullCodec, StorageValue: super::StorageValue<T>> StreamIterExt<T>
for StorageValue
{
type Iterator = ScaleContainerStreamIter<T::Stored>;

fn stream_iter() -> Self::Iterator {
ScaleContainerStreamIter::new(Self::storage_value_final_key().into()).unwrap()
fn stream_iter() -> T::Iterator {
T::stream_iter(Self::hashed_key().into())
}
}

Expand Down Expand Up @@ -113,17 +140,10 @@ impl<T: codec::Decode> sp_std::iter::Iterator for ScaleContainerStreamIter<T> {
}
}

impl<T: codec::Decode> StreamIter for Vec<T> {
type Iterator = ScaleContainerStreamIter<T>;

fn stream_iter(key: Vec<u8>) -> Self::Iterator {
ScaleContainerStreamIter::new(key).unwrap()
}
}

const STORAGE_INPUT_BUFFER_CAPACITY: usize = 16 * 1024;

pub struct StorageInput {
/// Implementation of [`codec::Input`]
struct StorageInput {
key: Vec<u8>,
offset: u32,
total_length: u32,
Expand All @@ -146,7 +166,7 @@ impl StorageInput {
(0, false)
};

if total_length < buffer.len() {
if (total_length as usize) < buffer.len() {
unsafe {
buffer.set_len(total_length as usize);
}
Expand Down Expand Up @@ -246,11 +266,11 @@ fn stream_read_test() {
let data: Vec<u32> = vec![1, 2, 3, 4, 5];
StreamReadTest::put(&data);

assert_eq!(data, StreamReadTest::stream().collect::<Vec<_>>());
assert_eq!(data, StreamReadTest::stream_iter().collect::<Vec<_>>());

let data: Vec<Vec<u8>> = vec![vec![0; 3000], vec![1; 2500]];
StreamReadTest2::put(&data);

assert_eq!(data, StreamReadTest2::stream().collect::<Vec<_>>());
assert_eq!(data, StreamReadTest2::stream_iter().collect::<Vec<_>>());
})
}
4 changes: 0 additions & 4 deletions frame/support/src/storage/types/value.rs
Original file line number Diff line number Diff line change
Expand Up @@ -199,10 +199,6 @@ where
{
<Self as crate::storage::TryAppendValue<Value, Item>>::try_append(item)
}

pub fn stream() -> <Value as crate::StreamRead>::Iterator where Value: crate::StreamRead {
<Value as crate::StreamRead>::stream(Self::hashed_key().into())
}
}

impl<Prefix, Value, QueryKind, OnEmpty> StorageEntryMetadataBuilder
Expand Down
4 changes: 2 additions & 2 deletions frame/system/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ use frame_support::{
extract_actual_pays_fee, extract_actual_weight, DispatchClass, DispatchInfo,
DispatchResult, DispatchResultWithPostInfo, PerDispatchClass,
},
storage,
storage::{self, stream_iter::StreamIterExt},
traits::{
ConstU32, Contains, EnsureOrigin, Get, HandleLifetime, OnKilledAccount, OnNewAccount,
OriginTrait, PalletInfo, SortedMembers, StoredMap, TypedGet,
Expand Down Expand Up @@ -1434,7 +1434,7 @@ impl<T: Config> Pallet<T> {
/// Should only be called if you know what you are doing and outside of the runtime block
/// execution else it can have a large impact on the PoV size of a block.
pub fn read_events_no_consensus() -> impl sp_std::iter::Iterator<Item = Box<EventRecord<T::RuntimeEvent, T::Hash>>> {
Events::<T>::stream()
Events::<T>::stream_iter()
}

/// Set the block number to something in particular. Can be used as an alternative to
Expand Down