From 7119fdcba707709e6a5dfd81cbb68c4236cea4b4 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 3 Feb 2020 11:20:39 +0100 Subject: [PATCH 1/9] Remove dead bootnodes, add new geth bootnodes --- ethcore/res/ethereum/foundation.json | 34 +++++++--------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/ethcore/res/ethereum/foundation.json b/ethcore/res/ethereum/foundation.json index ec864d6ed27..ba988e5bf40 100644 --- a/ethcore/res/ethereum/foundation.json +++ b/ethcore/res/ethereum/foundation.json @@ -4770,32 +4770,14 @@ "enode://4afb3a9137a88267c02651052cf6fb217931b8c78ee058bb86643542a4e2e0a8d24d47d871654e1b78a276c363f3c1bc89254a973b00adc359c9e9a48f140686@144.217.139.5:30303", "enode://c16d390b32e6eb1c312849fe12601412313165df1a705757d671296f1ac8783c5cff09eab0118ac1f981d7148c85072f0f26407e5c68598f3ad49209fade404d@139.99.51.203:30303", "enode://4faf867a2e5e740f9b874e7c7355afee58a2d1ace79f7b692f1d553a1134eddbeb5f9210dd14dc1b774a46fd5f063a8bc1fa90579e13d9d18d1f59bac4a4b16b@139.99.160.213:30303", - "enode://6a868ced2dec399c53f730261173638a93a40214cf299ccf4d42a76e3fa54701db410669e8006347a4b3a74fa090bb35af0320e4bc8d04cf5b7f582b1db285f5@163.172.131.191:30303", - "enode://66a483383882a518fcc59db6c017f9cd13c71261f13c8d7e67ed43adbbc82a932d88d2291f59be577e9425181fc08828dc916fdd053af935a9491edf9d6006ba@212.47.247.103:30303", - "enode://cd6611461840543d5b9c56fbf088736154c699c43973b3a1a32390cf27106f87e58a818a606ccb05f3866de95a4fe860786fea71bf891ea95f234480d3022aa3@163.172.157.114:30303", - "enode://1d1f7bcb159d308eb2f3d5e32dc5f8786d714ec696bb2f7e3d982f9bcd04c938c139432f13aadcaf5128304a8005e8606aebf5eebd9ec192a1471c13b5e31d49@138.201.223.35:30303", - "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", - "enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303", - "enode://78de8a0916848093c73790ead81d1928bec737d565119932b98c6b100d944b7a95e94f847f689fc723399d2e31129d182f7ef3863f2b4c820abbf3ab2722344d@191.235.84.50:30303", - "enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303", - "enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303", - "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303", - "enode://0cc5f5ffb5d9098c8b8c62325f3797f56509bff942704687b6530992ac706e2cb946b90a34f1f19548cd3c7baccbcaea354531e5983c7d1bc0dee16ce4b6440b@40.118.3.223:30305", - "enode://1c7a64d76c0334b0418c004af2f67c50e36a3be60b5e4790bdac0439d21603469a85fad36f2473c9a80eb043ae60936df905fa28f1ff614c3e5dc34f15dcd2dc@40.118.3.223:30308", - "enode://85c85d7143ae8bb96924f2b54f1b3e70d8c4d367af305325d30a61385a432f247d2c75c45c6b4a60335060d072d7f5b35dd1d4c45f76941f62a4f83b6e75daaf@40.118.3.223:30309", - "enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303", - "enode://4cd540b2c3292e17cff39922e864094bf8b0741fcc8c5dcea14957e389d7944c70278d872902e3d0345927f621547efa659013c400865485ab4bfa0c6596936f@138.201.144.135:30303", - "enode://01f76fa0561eca2b9a7e224378dd854278735f1449793c46ad0c4e79e8775d080c21dcc455be391e90a98153c3b05dcc8935c8440de7b56fe6d67251e33f4e3c@51.15.42.252:30303", - "enode://2c9059f05c352b29d559192fe6bca272d965c9f2290632a2cfda7f83da7d2634f3ec45ae3a72c54dd4204926fb8082dcf9686e0d7504257541c86fc8569bcf4b@163.172.171.38:30303", - "enode://efe4f2493f4aff2d641b1db8366b96ddacfe13e7a6e9c8f8f8cf49f9cdba0fdf3258d8c8f8d0c5db529f8123c8f1d95f36d54d590ca1bb366a5818b9a4ba521c@163.172.187.252:30303", - "enode://bcc7240543fe2cf86f5e9093d05753dd83343f8fda7bf0e833f65985c73afccf8f981301e13ef49c4804491eab043647374df1c4adf85766af88a624ecc3330e@136.243.154.244:30303", - "enode://ed4227681ca8c70beb2277b9e870353a9693f12e7c548c35df6bca6a956934d6f659999c2decb31f75ce217822eefca149ace914f1cbe461ed5a2ebaf9501455@88.212.206.70:30303", - "enode://cadc6e573b6bc2a9128f2f635ac0db3353e360b56deef239e9be7e7fce039502e0ec670b595f6288c0d2116812516ad6b6ff8d5728ff45eba176989e40dead1e@37.128.191.230:30303", - "enode://595a9a06f8b9bc9835c8723b6a82105aea5d55c66b029b6d44f229d6d135ac3ecdd3e9309360a961ea39d7bee7bac5d03564077a4e08823acc723370aace65ec@46.20.235.22:30303", - "enode://029178d6d6f9f8026fc0bc17d5d1401aac76ec9d86633bba2320b5eed7b312980c0a210b74b20c4f9a8b0b2bf884b111fa9ea5c5f916bb9bbc0e0c8640a0f56c@216.158.85.185:30303", - "enode://fdd1b9bb613cfbc200bba17ce199a9490edc752a833f88d4134bf52bb0d858aa5524cb3ec9366c7a4ef4637754b8b15b5dc913e4ed9fdb6022f7512d7b63f181@212.47.247.103:30303", - "enode://cc26c9671dffd3ee8388a7c8c5b601ae9fe75fc0a85cedb72d2dd733d5916fad1d4f0dcbebad5f9518b39cc1f96ba214ab36a7fa5103aaf17294af92a89f227b@52.79.241.155:30303", - "enode://140872ce4eee37177fbb7a3c3aa4aaebe3f30bdbf814dd112f6c364fc2e325ba2b6a942f7296677adcdf753c33170cb4999d2573b5ff7197b4c1868f25727e45@52.78.149.82:30303" + "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", + "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", + "enode://ca6de62fce278f96aea6ec5a2daadb877e51651247cb96ee310a318def462913b653963c155a0ef6c7d50048bba6e6cea881130857413d9f50a621546b590758@34.255.23.113:30303", + "enode://279944d8dcd428dffaa7436f25ca0ca43ae19e7bcf94a8fb7d1641651f92d121e972ac2e8f381414b80cc8e5555811c2ec6e1a99bb009b3f53c4c69923e11bd8@35.158.244.151:30303", + "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", + "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", + "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", + "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303" ], "accounts": { "0x0000000000000000000000000000000000000001": { From 145055133b66a9eb944645c3c3173e7fe277cc1e Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 4 Feb 2020 10:50:45 +0100 Subject: [PATCH 2/9] More granular locking when fetching state Finish GetDataNode requests early if queries take too long --- Cargo.lock | 7 +++++++ Cargo.toml | 2 +- ethcore/sync/Cargo.toml | 1 + ethcore/sync/src/chain/mod.rs | 5 +++++ ethcore/sync/src/chain/supplier.rs | 27 +++++++++++++++++++++------ util/journaldb/src/overlayrecentdb.rs | 17 +++++++++++++---- util/network-devp2p/src/connection.rs | 12 ++++++------ 7 files changed, 54 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index beadfb1f248..213318391d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -940,6 +940,11 @@ name = "either" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "elapsed" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "elastic-array" version = "0.10.2" @@ -1553,6 +1558,7 @@ version = "1.12.0" dependencies = [ "client-traits 0.1.0", "common-types 0.1.0", + "elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "engine 0.1.0", "enum_primitive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5671,6 +5677,7 @@ dependencies = [ "checksum docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "db2906c2579b5b7207fc1e328796a9a8835dc44e22dbe8e460b1d636f9a7b225" "checksum edit-distance 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3bd26878c3d921f89797a4e1a1711919f999a9f6946bb6f5a4ffda126d297b7e" "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" +"checksum elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f4e5af126dafd0741c2ad62d47f68b28602550102e5f0dd45c8a97fc8b49c29" "checksum elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "073be79b6538296faf81c631872676600616073817dd9a440c477ad09b408983" "checksum enum_primitive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180" "checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" diff --git a/Cargo.toml b/Cargo.toml index 9897a201274..e0f9c4990af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,7 +122,7 @@ path = "parity/main.rs" name = "parity" [profile.release] -debug = false +debug = true # todo[dvdplm] cleanup! lto = true [workspace] diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index 14e1d758bf7..7dc30b2f218 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -9,6 +9,7 @@ edition = "2018" [lib] [dependencies] +elapsed = "0.1.2" bytes = { package = "parity-bytes", version = "0.1" } client-traits = { path = "../client-traits" } common-types = { path = "../types" } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 8f9e296c036..6dd13afcd9e 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -162,7 +162,12 @@ pub const PAR_PROTOCOL_VERSION_4: (u8, u8) = (4, 0x20); pub const MAX_BODIES_TO_SEND: usize = 256; pub const MAX_HEADERS_TO_SEND: usize = 512; +/// Maximum number of "entries" to include in a GetDataNode request. pub const MAX_NODE_DATA_TO_SEND: usize = 1024; +/// Maximum allowed duration for serving a batch GetNodeData request. +const MAX_NODE_DATA_TOTAL_DURATION: Duration = Duration::from_millis(5_000); +/// Maximum allowed duration for serving a single GetNodeData request. +const MAX_NODE_DATA_SINGLE_DURATION: Duration = Duration::from_millis(50); pub const MAX_RECEIPTS_HEADERS_TO_SEND: usize = 256; const MIN_PEERS_PROPAGATION: usize = 4; const MAX_PEERS_PROPAGATION: usize = 128; diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 43e1148fc1e..57f5be610ea 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -15,10 +15,12 @@ // along with Parity Ethereum. If not, see . use std::cmp; +use std::time::Duration; use crate::sync_io::SyncIo; use bytes::Bytes; +use elapsed::measure_time; use enum_primitive::FromPrimitive; use ethereum_types::H256; use log::{debug, trace}; @@ -56,6 +58,8 @@ use super::{ MAX_BODIES_TO_SEND, MAX_HEADERS_TO_SEND, MAX_NODE_DATA_TO_SEND, + MAX_NODE_DATA_TOTAL_DURATION, + MAX_NODE_DATA_SINGLE_DURATION, MAX_RECEIPTS_HEADERS_TO_SEND, }; @@ -258,9 +262,9 @@ impl SyncSupplier { /// Respond to GetNodeData request fn return_node_data(io: &dyn SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { - let payload_soft_limit = io.payload_soft_limit(); + let payload_soft_limit = io.payload_soft_limit(); // 4Mb let mut count = r.item_count().unwrap_or(0); - trace!(target: "sync", "{} -> GetNodeData: {} entries", peer_id, count); + trace!(target: "sync_dp", "{} -> GetNodeData: {} entries requested", peer_id, count); if count == 0 { debug!(target: "sync", "Empty GetNodeData request, ignoring."); return Ok(None); @@ -269,10 +273,20 @@ impl SyncSupplier { let mut added = 0usize; let mut data = Vec::new(); let mut total_bytes = 0; + let mut total_elpsd = Duration::from_secs(0); for i in 0..count { - if let Some(node) = io.chain().state_data(&r.val_at::(i)?) { + let hash = &r.val_at(i)?; + let (elpsd, state) = measure_time(|| { + io.chain().state_data(hash) + }); + total_elpsd += elpsd.duration(); + if elpsd.duration() > MAX_NODE_DATA_SINGLE_DURATION || total_elpsd > MAX_NODE_DATA_TOTAL_DURATION { + trace!(target: "sync_dp", "{} -> GetNodeData: item {}/{} – slow state fetch for hash {:?}; took {}", + peer_id, i, count, hash, elpsd); + break; + } + if let Some(node) = state { total_bytes += node.len(); - // Check that the packet won't be oversized if total_bytes > payload_soft_limit { break; } @@ -280,7 +294,8 @@ impl SyncSupplier { added += 1; } } - trace!(target: "sync", "{} -> GetNodeData: return {} entries", peer_id, added); + trace!(target: "sync_dp", "{} -> GetNodeData: returning {}/{} entries ({} bytes total in {})", + peer_id, added, count, total_bytes, elapsed::ElapsedDuration::new(total_elpsd)); let mut rlp = RlpStream::new_list(added); for d in data { rlp.append(&d); @@ -540,7 +555,7 @@ mod test { let rlp_result = result.unwrap(); assert!(rlp_result.is_some()); - // the length of one rlp-encoded hashe + // the length of one rlp-encoded hash let rlp = rlp_result.unwrap().1.out(); let rlp = Rlp::new(&rlp); assert_eq!(Ok(1), rlp.item_count()); diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index ed4c5bb25bc..d4d05528f6a 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -20,6 +20,7 @@ use std::{ collections::{HashMap, hash_map::Entry}, io, sync::Arc, + time::Duration, }; use ethereum_types::H256; @@ -279,11 +280,19 @@ impl JournalDB for OverlayRecentDB { fn earliest_era(&self) -> Option { self.journal_overlay.read().earliest_era } fn state(&self, key: &H256) -> Option { - let journal_overlay = self.journal_overlay.read(); let key = to_short_key(key); - journal_overlay.backing_overlay.get(&key, EMPTY_PREFIX) - .or_else(|| journal_overlay.pending_overlay.get(&key).map(|d| d.clone())) - .or_else(|| self.backing.get_by_prefix(self.column, &key[0..DB_PREFIX_LEN]).map(|b| b.to_vec())) + let maybe_state_date = { + let journal_overlay = self.journal_overlay.try_read_for(Duration::from_secs(2))?; + journal_overlay.backing_overlay.get(&key, EMPTY_PREFIX) + .or_else(|| journal_overlay.pending_overlay.get(&key).map(|d| d.clone())) + }; + match maybe_state_date { + Some(data) => Some(data), + None => { + let pkey = &key[..DB_PREFIX_LEN]; + self.backing.get_by_prefix(self.column, &pkey).map(|b| b.to_vec()) + } + } } fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> io::Result { diff --git a/util/network-devp2p/src/connection.rs b/util/network-devp2p/src/connection.rs index 0ddeb3737dd..2f88efb2d09 100644 --- a/util/network-devp2p/src/connection.rs +++ b/util/network-devp2p/src/connection.rs @@ -40,11 +40,11 @@ use crate::handshake::Handshake; const ENCRYPTED_HEADER_LEN: usize = 32; const RECEIVE_PAYLOAD: Duration = Duration::from_secs(30); -pub const MAX_PAYLOAD_SIZE: usize = (1 << 24) - 1; +pub const MAX_PAYLOAD_SIZE: usize = (1 << 24) - 1; // 16Mb /// Network responses should try not to go over this limit. /// This should be lower than MAX_PAYLOAD_SIZE -pub const PAYLOAD_SOFT_LIMIT: usize = (1 << 22) - 1; +pub const PAYLOAD_SOFT_LIMIT: usize = (1 << 22) - 1; // 4Mb pub trait GenericSocket : Read + Write { } @@ -97,7 +97,7 @@ impl GenericConnection { else if self.rec_buf.len() > self.rec_size { warn!(target:"network", "Read past buffer {} bytes", self.rec_buf.len() - self.rec_size); return Ok(Some(::std::mem::replace(&mut self.rec_buf, Bytes::new()))) - } + } }, Ok(_) => return Ok(None), Err(e) => { @@ -105,7 +105,7 @@ impl GenericConnection { return Err(e) } } - } + } } /// Add a packet to send queue. @@ -222,7 +222,7 @@ impl Connection { pub fn register_socket(&self, reg: Token, event_loop: &mut EventLoop) -> io::Result<()> { if self.registered.compare_and_swap(false, true, AtomicOrdering::SeqCst) { return Ok(()); - } + } trace!(target: "network", "connection register; token={:?}", reg); if let Err(e) = event_loop.register(&self.socket, reg, self.interest, PollOpt::edge() /* | PollOpt::oneshot() */) { // TODO: oneshot is broken on windows trace!(target: "network", "Failed to register {:?}, {:?}", reg, e); @@ -235,7 +235,7 @@ impl Connection { trace!(target: "network", "connection reregister; token={:?}", reg); if !self.registered.load(AtomicOrdering::SeqCst) { self.register_socket(reg, event_loop) - } else { + } else { event_loop.reregister(&self.socket, reg, self.interest, PollOpt::edge() /* | PollOpt::oneshot() */ ).unwrap_or_else(|e| { // TODO: oneshot is broken on windows trace!(target: "network", "Failed to reregister {:?}, {:?}", reg, e); }); From 1e00f7d9a9445b708716b49ee64940e073a806fa Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 4 Feb 2020 11:12:45 +0100 Subject: [PATCH 3/9] typo --- util/journaldb/src/overlayrecentdb.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index d4d05528f6a..17203f7a4dd 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -281,12 +281,12 @@ impl JournalDB for OverlayRecentDB { fn state(&self, key: &H256) -> Option { let key = to_short_key(key); - let maybe_state_date = { + let maybe_state_data = { let journal_overlay = self.journal_overlay.try_read_for(Duration::from_secs(2))?; journal_overlay.backing_overlay.get(&key, EMPTY_PREFIX) .or_else(|| journal_overlay.pending_overlay.get(&key).map(|d| d.clone())) }; - match maybe_state_date { + match maybe_state_data { Some(data) => Some(data), None => { let pkey = &key[..DB_PREFIX_LEN]; From cf8c19a78fb1dc17e65af71f20055d8baf8021af Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 4 Feb 2020 13:50:33 +0100 Subject: [PATCH 4/9] Use latest kvdb-rocksdb --- Cargo.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c39f635d9a..fe275ab5718 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1125,7 +1125,7 @@ dependencies = [ "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-memorydb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "machine 0.1.0", @@ -1493,7 +1493,7 @@ dependencies = [ "jsonrpc-server-utils 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1528,7 +1528,7 @@ dependencies = [ "ethcore-sync 1.12.0", "ethereum-types 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "snapshot 0.1.0", "spec 0.1.0", @@ -2485,7 +2485,7 @@ dependencies = [ [[package]] name = "kvdb-rocksdb" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fs-swap 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2719,7 +2719,7 @@ name = "migration-rocksdb" version = "0.1.0" dependencies = [ "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "macros 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3104,7 +3104,7 @@ dependencies = [ "jsonrpc-core 14.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "migration-rocksdb 0.1.0", "node-filter 1.12.0", @@ -4401,7 +4401,7 @@ dependencies = [ "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4447,7 +4447,7 @@ dependencies = [ "keccak-hash 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5758,7 +5758,7 @@ dependencies = [ "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum kvdb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8396be0e5561ccd1bf7ff0b2007487cdd7a87a056873fe6ea906b35d4dbf7ed8" "checksum kvdb-memorydb 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4d25ef14155e418515c4839e9144c839de3506e68946f255a32b7f166095493d" -"checksum kvdb-rocksdb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "af488cc16c3801705c8d681c3a32c8faa8fafc7fb5309dee0f573f3c6a19d395" +"checksum kvdb-rocksdb 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5a1053e90a54421a842b6bf5d0e4a5cb5364c0bf570f713c58e44a9906f501d9" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum lazycell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddba4c30a78328befecec92fc94970e53b3ae385827d28620f0f5bb2493081e0" "checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" From 986b85ac26605284b97d80e708d354793fa11eac Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 4 Feb 2020 13:55:32 +0100 Subject: [PATCH 5/9] Cleanup --- Cargo.toml | 2 +- ethcore/sync/src/chain/supplier.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e0f9c4990af..9897a201274 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,7 +122,7 @@ path = "parity/main.rs" name = "parity" [profile.release] -debug = true # todo[dvdplm] cleanup! +debug = false lto = true [workspace] diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 57f5be610ea..3202b5db6db 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -264,7 +264,7 @@ impl SyncSupplier { fn return_node_data(io: &dyn SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { let payload_soft_limit = io.payload_soft_limit(); // 4Mb let mut count = r.item_count().unwrap_or(0); - trace!(target: "sync_dp", "{} -> GetNodeData: {} entries requested", peer_id, count); + trace!(target: "sync", "{} -> GetNodeData: {} entries requested", peer_id, count); if count == 0 { debug!(target: "sync", "Empty GetNodeData request, ignoring."); return Ok(None); @@ -281,7 +281,7 @@ impl SyncSupplier { }); total_elpsd += elpsd.duration(); if elpsd.duration() > MAX_NODE_DATA_SINGLE_DURATION || total_elpsd > MAX_NODE_DATA_TOTAL_DURATION { - trace!(target: "sync_dp", "{} -> GetNodeData: item {}/{} – slow state fetch for hash {:?}; took {}", + trace!(target: "sync", "{} -> GetNodeData: item {}/{} – slow state fetch for hash {:?}; took {}", peer_id, i, count, hash, elpsd); break; } @@ -294,7 +294,7 @@ impl SyncSupplier { added += 1; } } - trace!(target: "sync_dp", "{} -> GetNodeData: returning {}/{} entries ({} bytes total in {})", + trace!(target: "sync", "{} -> GetNodeData: returning {}/{} entries ({} bytes total in {})", peer_id, added, count, total_bytes, elapsed::ElapsedDuration::new(total_elpsd)); let mut rlp = RlpStream::new_list(added); for d in data { From 71a8f7dd89b7acff9345b51c999ff5010bf44991 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 4 Feb 2020 17:05:08 +0100 Subject: [PATCH 6/9] Update ethcore/sync/src/chain/supplier.rs Co-Authored-By: Andronik Ordian --- ethcore/sync/src/chain/supplier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 3202b5db6db..45edee5e95b 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -281,7 +281,7 @@ impl SyncSupplier { }); total_elpsd += elpsd.duration(); if elpsd.duration() > MAX_NODE_DATA_SINGLE_DURATION || total_elpsd > MAX_NODE_DATA_TOTAL_DURATION { - trace!(target: "sync", "{} -> GetNodeData: item {}/{} – slow state fetch for hash {:?}; took {}", + warn!(target: "sync", "{} -> GetNodeData: item {}/{} – slow state fetch for hash {:?}; took {}", peer_id, i, count, hash, elpsd); break; } From b439050b5953665a8485b4a5d3ea8a52d1b815d3 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 4 Feb 2020 17:21:18 +0100 Subject: [PATCH 7/9] Address review grumbles --- Cargo.lock | 7 ------- ethcore/sync/Cargo.toml | 1 - ethcore/sync/src/chain/supplier.rs | 19 +++++++++---------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe275ab5718..a283ea6949d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -939,11 +939,6 @@ name = "either" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "elapsed" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "elastic-array" version = "0.10.2" @@ -1557,7 +1552,6 @@ version = "1.12.0" dependencies = [ "client-traits 0.1.0", "common-types 0.1.0", - "elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "engine 0.1.0", "enum_primitive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5676,7 +5670,6 @@ dependencies = [ "checksum docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "db2906c2579b5b7207fc1e328796a9a8835dc44e22dbe8e460b1d636f9a7b225" "checksum edit-distance 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3bd26878c3d921f89797a4e1a1711919f999a9f6946bb6f5a4ffda126d297b7e" "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" -"checksum elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f4e5af126dafd0741c2ad62d47f68b28602550102e5f0dd45c8a97fc8b49c29" "checksum elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "073be79b6538296faf81c631872676600616073817dd9a440c477ad09b408983" "checksum enum_primitive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180" "checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index 7dc30b2f218..14e1d758bf7 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -9,7 +9,6 @@ edition = "2018" [lib] [dependencies] -elapsed = "0.1.2" bytes = { package = "parity-bytes", version = "0.1" } client-traits = { path = "../client-traits" } common-types = { path = "../types" } diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 3202b5db6db..29f12b0bf59 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -15,12 +15,11 @@ // along with Parity Ethereum. If not, see . use std::cmp; -use std::time::Duration; +use std::time::{Duration, Instant}; use crate::sync_io::SyncIo; use bytes::Bytes; -use elapsed::measure_time; use enum_primitive::FromPrimitive; use ethereum_types::H256; use log::{debug, trace}; @@ -276,12 +275,12 @@ impl SyncSupplier { let mut total_elpsd = Duration::from_secs(0); for i in 0..count { let hash = &r.val_at(i)?; - let (elpsd, state) = measure_time(|| { - io.chain().state_data(hash) - }); - total_elpsd += elpsd.duration(); - if elpsd.duration() > MAX_NODE_DATA_SINGLE_DURATION || total_elpsd > MAX_NODE_DATA_TOTAL_DURATION { - trace!(target: "sync", "{} -> GetNodeData: item {}/{} – slow state fetch for hash {:?}; took {}", + let elpsd = Instant::now(); + let state = io.chain().state_data(hash); + + total_elpsd += elpsd.elapsed(); + if elpsd.elapsed() > MAX_NODE_DATA_SINGLE_DURATION || total_elpsd > MAX_NODE_DATA_TOTAL_DURATION { + trace!(target: "sync", "{} -> GetNodeData: item {}/{} – slow state fetch for hash {:?}; took {:?}", peer_id, i, count, hash, elpsd); break; } @@ -294,8 +293,8 @@ impl SyncSupplier { added += 1; } } - trace!(target: "sync", "{} -> GetNodeData: returning {}/{} entries ({} bytes total in {})", - peer_id, added, count, total_bytes, elapsed::ElapsedDuration::new(total_elpsd)); + trace!(target: "sync", "{} -> GetNodeData: returning {}/{} entries ({} bytes total in {:?})", + peer_id, added, count, total_bytes, total_elpsd); let mut rlp = RlpStream::new_list(added); for d in data { rlp.append(&d); From 1d8c3e5e858229f6fd9556a14b5c98758d0f4994 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 4 Feb 2020 17:25:44 +0100 Subject: [PATCH 8/9] Fix compilation --- ethcore/sync/src/chain/supplier.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 577e89cec1c..6596506ed71 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -22,7 +22,7 @@ use crate::sync_io::SyncIo; use bytes::Bytes; use enum_primitive::FromPrimitive; use ethereum_types::H256; -use log::{debug, trace}; +use log::{debug, trace, warn}; use network::{self, PeerId}; use parking_lot::RwLock; use rlp::{Rlp, RlpStream}; From 358ec1e267ec38be854be7e303174fb1171591d6 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 4 Feb 2020 20:45:41 +0100 Subject: [PATCH 9/9] Address review grumbles --- ethcore/sync/src/chain/mod.rs | 4 ++-- util/journaldb/src/overlayrecentdb.rs | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 6dd13afcd9e..2b506c04f2e 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -165,9 +165,9 @@ pub const MAX_HEADERS_TO_SEND: usize = 512; /// Maximum number of "entries" to include in a GetDataNode request. pub const MAX_NODE_DATA_TO_SEND: usize = 1024; /// Maximum allowed duration for serving a batch GetNodeData request. -const MAX_NODE_DATA_TOTAL_DURATION: Duration = Duration::from_millis(5_000); +const MAX_NODE_DATA_TOTAL_DURATION: Duration = Duration::from_secs(2); /// Maximum allowed duration for serving a single GetNodeData request. -const MAX_NODE_DATA_SINGLE_DURATION: Duration = Duration::from_millis(50); +const MAX_NODE_DATA_SINGLE_DURATION: Duration = Duration::from_millis(100); pub const MAX_RECEIPTS_HEADERS_TO_SEND: usize = 256; const MIN_PEERS_PROPAGATION: usize = 4; const MAX_PEERS_PROPAGATION: usize = 128; diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index 17203f7a4dd..8f17637708b 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -281,18 +281,21 @@ impl JournalDB for OverlayRecentDB { fn state(&self, key: &H256) -> Option { let key = to_short_key(key); + // Hold the read lock for shortest possible amount of time. let maybe_state_data = { - let journal_overlay = self.journal_overlay.try_read_for(Duration::from_secs(2))?; - journal_overlay.backing_overlay.get(&key, EMPTY_PREFIX) + let journal_overlay = self.journal_overlay.read(); + journal_overlay + .backing_overlay + .get(&key, EMPTY_PREFIX) .or_else(|| journal_overlay.pending_overlay.get(&key).map(|d| d.clone())) }; - match maybe_state_data { - Some(data) => Some(data), - None => { - let pkey = &key[..DB_PREFIX_LEN]; - self.backing.get_by_prefix(self.column, &pkey).map(|b| b.to_vec()) - } - } + + maybe_state_data.or_else(|| { + let pkey = &key[..DB_PREFIX_LEN]; + self.backing + .get_by_prefix(self.column, &pkey) + .map(|b| b.to_vec()) + }) } fn journal_under(&mut self, batch: &mut DBTransaction, now: u64, id: &H256) -> io::Result {