Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

[Testing PR] Testing batch dispute import + vote prioritisation #6010

Closed
wants to merge 66 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
66 commits
Select commit Hold shift + click to select a range
b202cc8
Start work on batching in dispute-distribution.
eskimor Aug 16, 2022
192d73b
Guide work.
eskimor Aug 17, 2022
a2f8fde
More guide changes. Still very much WIP.
eskimor Aug 18, 2022
5e2f4a5
Finish guide changes.
eskimor Aug 22, 2022
585ec6d
Clarification
eskimor Aug 22, 2022
7e02910
Adjust argument about slashing.
eskimor Aug 23, 2022
4e170e2
WIP: Add constants to receiver.
eskimor Aug 23, 2022
362ebae
Maintain order of disputes.
eskimor Aug 24, 2022
68c7073
dispute-distribuion sender Rate limit.
eskimor Aug 24, 2022
56519a7
Cleanup
eskimor Aug 25, 2022
b3ab280
WIP: dispute-distribution receiver.
eskimor Aug 25, 2022
4b4df00
WIP: Batching.
eskimor Aug 25, 2022
5816c8d
fmt
eskimor Aug 25, 2022
a821efc
Update `PeerQueues` to maintain more invariants.
eskimor Aug 26, 2022
3b71817
WIP: Batching.
eskimor Aug 26, 2022
909569b
Small cleanup
eskimor Aug 26, 2022
fa14c43
Batching logic.
eskimor Aug 30, 2022
89d622d
Some integration work.
eskimor Aug 30, 2022
855d6da
Add `DisputeState` to `DisputeCoordinatorMessage::RecentDisputes`
tdimitrov May 14, 2022
396187e
Move dummy_signature() in primitives/test-helpers
tdimitrov May 11, 2022
23013f4
Enable staging runtime api on Rococo
tdimitrov May 25, 2022
140c1e8
Implementation
tdimitrov May 16, 2022
040d1fb
Fix staging api usage
tdimitrov Aug 24, 2022
b8f6566
Fix `get_disputes` runtime function implementation
tdimitrov Aug 24, 2022
1be7983
Fix compilation error
tdimitrov Aug 24, 2022
60c1ddb
Fix arithmetic operations in tests
tdimitrov Aug 25, 2022
e1654e7
Use smaller test data
tdimitrov Aug 25, 2022
e3179cb
Rename `RuntimeApiRequest::StagingDisputes` to `RuntimeApiRequest::Di…
tdimitrov Aug 25, 2022
7e3bab7
Remove `staging-client` feature flag
tdimitrov Aug 25, 2022
24ae8ce
fmt
tdimitrov Aug 25, 2022
00bd01b
Remove `vstaging` feature flag
tdimitrov Aug 25, 2022
c771aa6
Some comments regarding the staging api
tdimitrov Aug 25, 2022
543b4ef
Rename dispute selection modules in provisioner
tdimitrov Aug 25, 2022
98ee76f
Comments for staging api
tdimitrov Aug 25, 2022
1884262
Comments
tdimitrov Aug 25, 2022
e6a6843
Additional logging
tdimitrov Aug 25, 2022
5f8b462
Code review feedback
tdimitrov Aug 29, 2022
95b0096
Code review feedback
tdimitrov Aug 30, 2022
0f9cf6a
Fix metrics
tdimitrov Aug 30, 2022
308a363
get_disputes -> disputes
tdimitrov Aug 30, 2022
a969eba
Get time only once during partitioning
tdimitrov Aug 30, 2022
57e1b88
Fix partitioning
tdimitrov Aug 30, 2022
db34125
Comments
tdimitrov Aug 31, 2022
d2af967
Reduce the number of hardcoded api versions
tdimitrov Aug 31, 2022
401ffb8
Finish.
eskimor Aug 31, 2022
1366d90
Code review feedback
tdimitrov Aug 31, 2022
3dd2041
Typo.
eskimor Sep 1, 2022
aeef116
Unused import
tdimitrov Sep 1, 2022
f9fc6c8
Docs.
eskimor Sep 1, 2022
43d946d
Report missing metric.
eskimor Sep 1, 2022
3e5ceaa
Doc pass.
eskimor Sep 1, 2022
0e11f4f
Tests for waiting_queue.
eskimor Sep 2, 2022
2fcebad
Speed up some crypto by 10x.
eskimor Sep 2, 2022
3d444ae
Fix redundant import.
eskimor Sep 2, 2022
e7923a7
Add some tracing.
eskimor Sep 2, 2022
8b8f722
Better sender rate limit
eskimor Sep 2, 2022
a79a96d
Some tests.
eskimor Sep 2, 2022
fd24899
Comments
tdimitrov Sep 5, 2022
c1ed615
Tests
eskimor Sep 5, 2022
f4c530c
Add logging to rate limiter
eskimor Sep 5, 2022
795c6c1
Update roadmap/implementers-guide/src/node/disputes/dispute-distribut…
eskimor Sep 8, 2022
d46be76
Update roadmap/implementers-guide/src/node/disputes/dispute-distribut…
eskimor Sep 8, 2022
9b8787d
Update node/network/dispute-distribution/src/receiver/mod.rs
eskimor Sep 8, 2022
9795098
Review feedback.
eskimor Sep 9, 2022
49a7b24
More precise log messages
tdimitrov Sep 14, 2022
25e1ee8
Merge remote-tracking branch 'upstream/rk-batch-vote-import' into mis…
tdimitrov Sep 14, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use smaller test data
  • Loading branch information
tdimitrov committed Aug 31, 2022
commit e1654e7702bc83b065bb21da8eaf1a2c99cef67e
6 changes: 6 additions & 0 deletions node/core/provisioner/src/disputes/with_staging_api/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,10 @@ mod tests;

/// The maximum number of disputes Provisioner will include in the inherent data.
/// Serves as a protection not to flood the Runtime with excessive data.
#[cfg(not(test))]
pub const MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME: usize = 200 * 1_000;
#[cfg(test)]
pub const MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME: usize = 200;
// The magic numbers are: `estimated validators count` * `estimated disputes per validator`

/// Implements the `select_disputes` function which selects dispute votes which should
Expand Down Expand Up @@ -153,7 +156,10 @@ async fn vote_selection<Sender>(
where
Sender: overseer::ProvisionerSenderTrait,
{
#[cfg(not(test))]
const BATCH_SIZE: usize = 1_100;
#[cfg(test)]
const BATCH_SIZE: usize = 11;

// fetch in batches until there are enough votes
let mut disputes = partitioned.into_iter().collect::<Vec<_>>();
Expand Down
23 changes: 13 additions & 10 deletions node/core/provisioner/src/disputes/with_staging_api/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,7 @@ impl TestDisputes {
&mut self,
dispute_count: usize,
) -> (u32, usize) {
let local_votes_count = self.validators_count * 70 / 100;
let local_votes_count = self.validators_count * 90 / 100;
let session_idx = 2;
let lf = leaf();
let dummy_receipt = test_helpers::dummy_candidate_receipt(lf.hash.clone());
Expand All @@ -427,7 +427,7 @@ impl TestDisputes {
}

pub fn add_concluded_disputes_known_onchain(&mut self, dispute_count: usize) -> (u32, usize) {
let local_votes_count = self.validators_count * 80 / 100;
let local_votes_count = self.validators_count * 90 / 100;
let onchain_votes_count = self.validators_count * 75 / 100;
let session_idx = 3;
let lf = leaf();
Expand All @@ -441,7 +441,7 @@ impl TestDisputes {
}

pub fn add_concluded_disputes_unknown_onchain(&mut self, dispute_count: usize) -> (u32, usize) {
let local_votes_count = self.validators_count * 80 / 100;
let local_votes_count = self.validators_count * 90 / 100;
let session_idx = 4;
let lf = leaf();
let dummy_receipt = test_helpers::dummy_candidate_receipt(lf.hash.clone());
Expand Down Expand Up @@ -486,8 +486,8 @@ impl TestDisputes {

#[test]
fn normal_flow() {
const VALIDATOR_COUNT: usize = 100;
const DISPUTES_PER_BATCH: usize = 10;
const VALIDATOR_COUNT: usize = 10;
const DISPUTES_PER_BATCH: usize = 2;
const ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT: usize = 1;

let mut input = TestDisputes::new(VALIDATOR_COUNT);
Expand Down Expand Up @@ -565,9 +565,12 @@ fn normal_flow() {

#[test]
fn many_batches() {
const VALIDATOR_COUNT: usize = 100;
const DISPUTES_PER_PARTITION: usize = 1000;
// Around 4_000 disputes are generated. `BATCH_SIZE` is 1_100.
const VALIDATOR_COUNT: usize = 10;
const DISPUTES_PER_PARTITION: usize = 10;

// 10 disputes per partition * 4 partitions = 40 disputes
// BATCH_SIZE = 11
// => There should be no more than 40 / 11 queries ( ~4 )
const ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT: usize = 4;

let mut input = TestDisputes::new(VALIDATOR_COUNT);
Expand Down Expand Up @@ -618,8 +621,8 @@ fn many_batches() {

#[test]
fn votes_above_limit() {
const VALIDATOR_COUNT: usize = 100;
const DISPUTES_PER_PARTITION: usize = 5_000;
const VALIDATOR_COUNT: usize = 10;
const DISPUTES_PER_PARTITION: usize = 50;
const ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT: usize = 4;

let mut input = TestDisputes::new(VALIDATOR_COUNT);
Expand Down