From b7c98e26a5222ae50aa14721d322615753dabb84 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 29 Jul 2016 11:23:31 +0200 Subject: [PATCH 1/6] removed configure_cache method --- ethcore/src/blockchain/blockchain.rs | 25 +++++++++---------------- ethcore/src/client/client.rs | 6 +----- ethcore/src/trace/db.rs | 4 ++++ 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 8b120caa2a7..1cfa3215f10 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -148,8 +148,8 @@ impl bc::group::BloomGroupDatabase for BlockChain { /// **Does not do input data verification.** pub struct BlockChain { // All locks must be captured in the order declared here. - pref_cache_size: AtomicUsize, - max_cache_size: AtomicUsize, + pref_cache_size: usize, + max_cache_size: usize, blooms_config: bc::Config, best_block: RwLock, @@ -324,8 +324,8 @@ impl BlockChain { (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); let bc = BlockChain { - pref_cache_size: AtomicUsize::new(config.pref_cache_size), - max_cache_size: AtomicUsize::new(config.max_cache_size), + pref_cache_size: config.pref_cache_size, + max_cache_size: config.max_cache_size, blooms_config: bc::Config { levels: LOG_BLOOMS_LEVELS, elements_per_index: LOG_BLOOMS_ELEMENTS_PER_INDEX, @@ -449,12 +449,6 @@ impl BlockChain { None } - /// Set the cache configuration. - pub fn configure_cache(&self, pref_cache_size: usize, max_cache_size: usize) { - self.pref_cache_size.store(pref_cache_size, AtomicOrder::Relaxed); - self.max_cache_size.store(max_cache_size, AtomicOrder::Relaxed); - } - /// Returns a tree route between `from` and `to`, which is a tuple of: /// /// - a vector of hashes of all blocks, ordered from `from` to `to`. @@ -888,11 +882,11 @@ impl BlockChain { /// Ticks our cache system and throws out any old data. pub fn collect_garbage(&self) { - if self.cache_size().total() < self.pref_cache_size.load(AtomicOrder::Relaxed) { + if self.cache_size().total() < self.pref_cache_size { // rotate cache let mut cache_man = self.cache_man.write(); const AVERAGE_BYTES_PER_CACHE_ENTRY: usize = 400; //estimated - if cache_man.cache_usage[0].len() > self.pref_cache_size.load(AtomicOrder::Relaxed) / COLLECTION_QUEUE_SIZE / AVERAGE_BYTES_PER_CACHE_ENTRY { + if cache_man.cache_usage[0].len() > self.pref_cache_size / COLLECTION_QUEUE_SIZE / AVERAGE_BYTES_PER_CACHE_ENTRY { trace!("Cache rotation, cache_size = {}", self.cache_size().total()); let cache = cache_man.cache_usage.pop_back().unwrap(); cache_man.cache_usage.push_front(cache); @@ -926,9 +920,6 @@ impl BlockChain { } cache_man.cache_usage.push_front(HashSet::new()); - // TODO: handle block_hashes properly. - block_hashes.clear(); - block_headers.shrink_to_fit(); block_bodies.shrink_to_fit(); block_details.shrink_to_fit(); @@ -938,7 +929,9 @@ impl BlockChain { block_receipts.shrink_to_fit(); } trace!("Cache cleanup round complete {}, cache_size = {}", i, self.cache_size().total()); - if self.cache_size().total() < self.max_cache_size.load(AtomicOrder::Relaxed) { break; } + if self.cache_size().total() < self.max_cache_size { + break; + } } // TODO: m_lastCollection = chrono::system_clock::now(); diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 3ff6b97336d..8ff0ea856c3 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -548,6 +548,7 @@ impl Client { pub fn tick(&self) { self.chain.collect_garbage(); self.block_queue.collect_garbage(); + self.tracedb.collect_garbage(); match self.mode { Mode::Dark(timeout) => { @@ -581,11 +582,6 @@ impl Client { } } - /// Set up the cache behaviour. - pub fn configure_cache(&self, pref_cache_size: usize, max_cache_size: usize) { - self.chain.configure_cache(pref_cache_size, max_cache_size); - } - /// Look up the block number for the given block ID. pub fn block_number(&self, id: BlockID) -> Option { match id { diff --git a/ethcore/src/trace/db.rs b/ethcore/src/trace/db.rs index c0dab5d1787..f7f73663b9e 100644 --- a/ethcore/src/trace/db.rs +++ b/ethcore/src/trace/db.rs @@ -145,6 +145,10 @@ impl TraceDB where T: DatabaseExtras { Ok(db) } + /// Ticks our cache system and throws out any old data. + pub fn collect_garbage(&self) { + } + /// Returns traces for block with hash. fn traces(&self, block_hash: &H256) -> Option { self.tracesdb.read_with_cache(DB_COL_TRACE, &self.traces, block_hash) From 0321cac7d708696da0353420b92d2087e8d78c43 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 29 Jul 2016 12:10:58 +0200 Subject: [PATCH 2/6] generic cache_manager struct --- ethcore/src/cache_manager.rs | 55 ++++++++++++++++++++++++++++++++++++ ethcore/src/lib.rs | 1 + 2 files changed, 56 insertions(+) create mode 100644 ethcore/src/cache_manager.rs diff --git a/ethcore/src/cache_manager.rs b/ethcore/src/cache_manager.rs new file mode 100644 index 00000000000..d24446827f1 --- /dev/null +++ b/ethcore/src/cache_manager.rs @@ -0,0 +1,55 @@ +use std::collections::{VecDeque, HashSet}; +use std::hash::Hash; + +const COLLECTION_QUEUE_SIZE: usize = 8; + +pub struct CacheManager where T: Eq + Hash { + pref_cache_size: usize, + max_cache_size: usize, + bytes_per_cache_entry: usize, + cache_usage: VecDeque> +} + +impl CacheManager where T: Eq + Hash { + pub fn new(pref_cache_size: usize, max_cache_size: usize, bytes_per_cache_entry: usize) -> Self { + CacheManager { + pref_cache_size: pref_cache_size, + max_cache_size: max_cache_size, + bytes_per_cache_entry: bytes_per_cache_entry, + cache_usage: (0..COLLECTION_QUEUE_SIZE).into_iter().map(|_| Default::default()).collect(), + } + } + + pub fn note_used(&mut self, id: T) { + if !self.cache_usage[0].contains(&id) { + if let Some(c) = self.cache_usage.iter_mut().skip(1).find(|e| e.contains(&id)) { + c.remove(&id); + } + self.cache_usage[0].insert(id); + } + } + + pub fn collect_carbage(&mut self, current_size: C, notify_unused: F) where C: Fn() -> usize, F: Fn(T) { + if current_size() < self.pref_cache_size { + self.rotate_cache_if_needed(); + return; + } + + for i in 0..COLLECTION_QUEUE_SIZE { + for id in self.cache_usage.pop_back().unwrap().into_iter() { + notify_unused(id) + } + self.cache_usage.push_front(Default::default()); + if current_size() < self.max_cache_size { + break; + } + } + } + + fn rotate_cache_if_needed(&mut self) { + if self.cache_usage[0].len() * self.bytes_per_cache_entry > self.pref_cache_size / COLLECTION_QUEUE_SIZE { + let cache = self.cache_usage.pop_back().unwrap(); + self.cache_usage.push_front(cache); + } + } +} diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 3c6f319a12f..2bcabab053e 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -117,6 +117,7 @@ pub mod snapshot; pub mod action_params; #[macro_use] pub mod evm; +mod cache_manager; mod blooms; mod db; mod common; From 2e23df1cbafb8a40ee73af3f9f844195adfb420d Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 29 Jul 2016 13:16:42 +0200 Subject: [PATCH 3/6] fixed #1743, tracing caches are cleared --- ethcore/src/blockchain/blockchain.rs | 106 ++++++++------------------- ethcore/src/cache_manager.rs | 24 ++++-- ethcore/src/trace/config.rs | 9 ++- ethcore/src/trace/db.rs | 39 +++++++++- 4 files changed, 94 insertions(+), 84 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 1cfa3215f10..cc80b7ef0a8 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -16,7 +16,6 @@ //! Blockchain database. -use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrder}; use bloomchain as bc; use util::*; use header::*; @@ -32,6 +31,7 @@ use blockchain::update::ExtrasUpdate; use blockchain::{CacheSize, ImportRoute, Config}; use db::{Writable, Readable, CacheUpdatePolicy}; use client::{DB_COL_EXTRA, DB_COL_HEADERS, DB_COL_BODIES}; +use cache_manager::CacheManager; const LOG_BLOOMS_LEVELS: usize = 3; const LOG_BLOOMS_ELEMENTS_PER_INDEX: usize = 16; @@ -130,11 +130,6 @@ enum CacheID { BlockReceipts(H256), } -struct CacheManager { - cache_usage: VecDeque>, - in_use: HashSet, -} - impl bc::group::BloomGroupDatabase for BlockChain { fn blooms_at(&self, position: &bc::group::GroupPosition) -> Option { let position = LogGroupPosition::from(position.clone()); @@ -148,8 +143,6 @@ impl bc::group::BloomGroupDatabase for BlockChain { /// **Does not do input data verification.** pub struct BlockChain { // All locks must be captured in the order declared here. - pref_cache_size: usize, - max_cache_size: usize, blooms_config: bc::Config, best_block: RwLock, @@ -167,7 +160,7 @@ pub struct BlockChain { db: Arc, - cache_man: RwLock, + cache_man: RwLock>, } impl BlockProvider for BlockChain { @@ -297,8 +290,6 @@ impl BlockProvider for BlockChain { } } -const COLLECTION_QUEUE_SIZE: usize = 8; - pub struct AncestryIter<'a> { current: H256, chain: &'a BlockChain, @@ -320,12 +311,10 @@ impl<'a> Iterator for AncestryIter<'a> { impl BlockChain { /// Create new instance of blockchain from given Genesis pub fn new(config: Config, genesis: &[u8], db: Arc) -> BlockChain { - let mut cache_man = CacheManager{cache_usage: VecDeque::new(), in_use: HashSet::new()}; - (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); + // 400 is the avarage size of the key + let cache_man = CacheManager::new(config.pref_cache_size, config.max_cache_size, 400); let bc = BlockChain { - pref_cache_size: config.pref_cache_size, - max_cache_size: config.max_cache_size, blooms_config: bc::Config { levels: LOG_BLOOMS_LEVELS, elements_per_index: LOG_BLOOMS_ELEMENTS_PER_INDEX, @@ -868,73 +857,40 @@ impl BlockChain { /// Let the cache system know that a cacheable item has been used. fn note_used(&self, id: CacheID) { let mut cache_man = self.cache_man.write(); - if !cache_man.cache_usage[0].contains(&id) { - cache_man.cache_usage[0].insert(id.clone()); - if cache_man.in_use.contains(&id) { - if let Some(c) = cache_man.cache_usage.iter_mut().skip(1).find(|e|e.contains(&id)) { - c.remove(&id); - } - } else { - cache_man.in_use.insert(id); - } - } + cache_man.note_used(id); } /// Ticks our cache system and throws out any old data. pub fn collect_garbage(&self) { - if self.cache_size().total() < self.pref_cache_size { - // rotate cache - let mut cache_man = self.cache_man.write(); - const AVERAGE_BYTES_PER_CACHE_ENTRY: usize = 400; //estimated - if cache_man.cache_usage[0].len() > self.pref_cache_size / COLLECTION_QUEUE_SIZE / AVERAGE_BYTES_PER_CACHE_ENTRY { - trace!("Cache rotation, cache_size = {}", self.cache_size().total()); - let cache = cache_man.cache_usage.pop_back().unwrap(); - cache_man.cache_usage.push_front(cache); - } - return; - } + let mut block_headers = self.block_headers.write(); + let mut block_bodies = self.block_bodies.write(); + let mut block_details = self.block_details.write(); + let mut block_hashes = self.block_hashes.write(); + let mut transaction_addresses = self.transaction_addresses.write(); + let mut blocks_blooms = self.blocks_blooms.write(); + let mut block_receipts = self.block_receipts.write(); - for i in 0..COLLECTION_QUEUE_SIZE { - { - trace!("Cache cleanup round started {}, cache_size = {}", i, self.cache_size().total()); - let mut block_headers = self.block_headers.write(); - let mut block_bodies = self.block_bodies.write(); - let mut block_details = self.block_details.write(); - let mut block_hashes = self.block_hashes.write(); - let mut transaction_addresses = self.transaction_addresses.write(); - let mut blocks_blooms = self.blocks_blooms.write(); - let mut block_receipts = self.block_receipts.write(); - let mut cache_man = self.cache_man.write(); - - for id in cache_man.cache_usage.pop_back().unwrap().into_iter() { - cache_man.in_use.remove(&id); - match id { - CacheID::BlockHeader(h) => { block_headers.remove(&h); }, - CacheID::BlockBody(h) => { block_bodies.remove(&h); }, - CacheID::BlockDetails(h) => { block_details.remove(&h); } - CacheID::BlockHashes(h) => { block_hashes.remove(&h); } - CacheID::TransactionAddresses(h) => { transaction_addresses.remove(&h); } - CacheID::BlocksBlooms(h) => { blocks_blooms.remove(&h); } - CacheID::BlockReceipts(h) => { block_receipts.remove(&h); } - } + let mut cache_man = self.cache_man.write(); + cache_man.collect_carbage(|| self.cache_size().total(), | ids | { + for id in &ids { + match *id { + CacheID::BlockHeader(ref h) => { block_headers.remove(h); }, + CacheID::BlockBody(ref h) => { block_bodies.remove(h); }, + CacheID::BlockDetails(ref h) => { block_details.remove(h); } + CacheID::BlockHashes(ref h) => { block_hashes.remove(h); } + CacheID::TransactionAddresses(ref h) => { transaction_addresses.remove(h); } + CacheID::BlocksBlooms(ref h) => { blocks_blooms.remove(h); } + CacheID::BlockReceipts(ref h) => { block_receipts.remove(h); } } - cache_man.cache_usage.push_front(HashSet::new()); - - block_headers.shrink_to_fit(); - block_bodies.shrink_to_fit(); - block_details.shrink_to_fit(); - block_hashes.shrink_to_fit(); - transaction_addresses.shrink_to_fit(); - blocks_blooms.shrink_to_fit(); - block_receipts.shrink_to_fit(); - } - trace!("Cache cleanup round complete {}, cache_size = {}", i, self.cache_size().total()); - if self.cache_size().total() < self.max_cache_size { - break; } - } - - // TODO: m_lastCollection = chrono::system_clock::now(); + block_headers.shrink_to_fit(); + block_bodies.shrink_to_fit(); + block_details.shrink_to_fit(); + block_hashes.shrink_to_fit(); + transaction_addresses.shrink_to_fit(); + blocks_blooms.shrink_to_fit(); + block_receipts.shrink_to_fit(); + }); } /// Create a block body from a block. diff --git a/ethcore/src/cache_manager.rs b/ethcore/src/cache_manager.rs index d24446827f1..b132d78aa80 100644 --- a/ethcore/src/cache_manager.rs +++ b/ethcore/src/cache_manager.rs @@ -1,3 +1,19 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + use std::collections::{VecDeque, HashSet}; use std::hash::Hash; @@ -29,16 +45,14 @@ impl CacheManager where T: Eq + Hash { } } - pub fn collect_carbage(&mut self, current_size: C, notify_unused: F) where C: Fn() -> usize, F: Fn(T) { + pub fn collect_carbage(&mut self, current_size: C, mut notify_unused: F) where C: Fn() -> usize, F: FnMut(HashSet) { if current_size() < self.pref_cache_size { self.rotate_cache_if_needed(); return; } - for i in 0..COLLECTION_QUEUE_SIZE { - for id in self.cache_usage.pop_back().unwrap().into_iter() { - notify_unused(id) - } + for _ in 0..COLLECTION_QUEUE_SIZE { + notify_unused(self.cache_usage.pop_back().unwrap()); self.cache_usage.push_front(Default::default()); if current_size() < self.max_cache_size { break; diff --git a/ethcore/src/trace/config.rs b/ethcore/src/trace/config.rs index 1c464681707..9a9db9cf9a3 100644 --- a/ethcore/src/trace/config.rs +++ b/ethcore/src/trace/config.rs @@ -68,8 +68,10 @@ pub struct Config { pub enabled: Switch, /// Traces blooms configuration. pub blooms: BloomConfig, - /// Database cache-size if not default - pub db_cache_size: Option, + /// Preferef cache-size. + pub pref_cache_size: usize, + /// Max cache-size. + pub max_cache_size: usize, } impl Default for Config { @@ -80,7 +82,8 @@ impl Default for Config { levels: 3, elements_per_index: 16, }, - db_cache_size: None, + pref_cache_size: 3 * 1024 * 1024, + max_cache_size: 5 * 1024 * 1024, } } } diff --git a/ethcore/src/trace/db.rs b/ethcore/src/trace/db.rs index f7f73663b9e..3b3384144c7 100644 --- a/ethcore/src/trace/db.rs +++ b/ethcore/src/trace/db.rs @@ -27,7 +27,7 @@ use db::{Key, Writable, Readable, CacheUpdatePolicy}; use blooms; use super::flat::{FlatTrace, FlatBlockTraces, FlatTransactionTraces}; use client::DB_COL_TRACE; - +use cache_manager::CacheManager; const TRACE_DB_VER: &'static [u8] = b"1.0"; @@ -88,11 +88,18 @@ impl Key for TraceGroupPosition { } } +#[derive(Debug, Hash, Eq, PartialEq)] +enum CacheID { + Trace(H256), + Bloom(TraceGroupPosition), +} + /// Trace database. pub struct TraceDB where T: DatabaseExtras { // cache traces: RwLock>, blooms: RwLock>, + cache_manager: RwLock>, // db tracesdb: Arc, // config, @@ -106,6 +113,7 @@ pub struct TraceDB where T: DatabaseExtras { impl BloomGroupDatabase for TraceDB where T: DatabaseExtras { fn blooms_at(&self, position: &GroupPosition) -> Option { let position = TraceGroupPosition::from(position.clone()); + self.note_used(CacheID::Bloom(position.clone())); self.tracesdb.read_with_cache(DB_COL_TRACE, &self.blooms, &position).map(Into::into) } } @@ -136,6 +144,7 @@ impl TraceDB where T: DatabaseExtras { let db = TraceDB { traces: RwLock::new(HashMap::new()), blooms: RwLock::new(HashMap::new()), + cache_manager: RwLock::new(CacheManager::new(config.pref_cache_size, config.max_cache_size, 10 * 1024)), tracesdb: tracesdb, bloom_config: config.blooms, enabled: enabled, @@ -145,12 +154,36 @@ impl TraceDB where T: DatabaseExtras { Ok(db) } + fn cache_size(&self) -> usize { + 0 + } + + /// Let the cache system know that a cacheable item has been used. + fn note_used(&self, id: CacheID) { + let mut cache_manager = self.cache_manager.write(); + cache_manager.note_used(id); + } + /// Ticks our cache system and throws out any old data. pub fn collect_garbage(&self) { + let mut traces = self.traces.write(); + let mut blooms = self.blooms.write(); + let mut cache_manager = self.cache_manager.write(); + cache_manager.collect_carbage(|| self.cache_size(), | ids | { + for id in &ids { + match *id { + CacheID::Trace(ref h) => { traces.remove(h); }, + CacheID::Bloom(ref h) => { blooms.remove(h); }, + } + } + traces.shrink_to_fit(); + blooms.shrink_to_fit(); + }); } /// Returns traces for block with hash. fn traces(&self, block_hash: &H256) -> Option { + self.note_used(CacheID::Trace(block_hash.clone())); self.tracesdb.read_with_cache(DB_COL_TRACE, &self.traces, block_hash) } @@ -225,6 +258,7 @@ impl TraceDatabase for TraceDB where T: DatabaseExtras { let mut traces = self.traces.write(); // it's important to use overwrite here, // cause this value might be queried by hash later + self.note_used(CacheID::Trace(request.block_hash.clone())); batch.write_with_cache(DB_COL_TRACE, traces.deref_mut(), request.block_hash, request.traces, CacheUpdatePolicy::Overwrite); } @@ -251,6 +285,9 @@ impl TraceDatabase for TraceDB where T: DatabaseExtras { .collect::>(); let mut blooms = self.blooms.write(); + for key in blooms_to_insert.keys() { + self.note_used(CacheID::Bloom(key.clone())); + } batch.extend_with_cache(DB_COL_TRACE, blooms.deref_mut(), blooms_to_insert, CacheUpdatePolicy::Remove); } } From c3c092613496029fceb20c3744f720f8a600940d Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 29 Jul 2016 13:38:33 +0200 Subject: [PATCH 4/6] removed deadlocks in garbage_collect, implemented HeapSizeOf for traces --- ethcore/src/blockchain/blockchain.rs | 16 ++++++++-------- ethcore/src/trace/db.rs | 17 +++++++++++++---- ethcore/src/types/trace_types/flat.rs | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index cc80b7ef0a8..abd42bc7f73 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -862,16 +862,16 @@ impl BlockChain { /// Ticks our cache system and throws out any old data. pub fn collect_garbage(&self) { - let mut block_headers = self.block_headers.write(); - let mut block_bodies = self.block_bodies.write(); - let mut block_details = self.block_details.write(); - let mut block_hashes = self.block_hashes.write(); - let mut transaction_addresses = self.transaction_addresses.write(); - let mut blocks_blooms = self.blocks_blooms.write(); - let mut block_receipts = self.block_receipts.write(); - let mut cache_man = self.cache_man.write(); cache_man.collect_carbage(|| self.cache_size().total(), | ids | { + let mut block_headers = self.block_headers.write(); + let mut block_bodies = self.block_bodies.write(); + let mut block_details = self.block_details.write(); + let mut block_hashes = self.block_hashes.write(); + let mut transaction_addresses = self.transaction_addresses.write(); + let mut blocks_blooms = self.blocks_blooms.write(); + let mut block_receipts = self.block_receipts.write(); + for id in &ids { match *id { CacheID::BlockHeader(ref h) => { block_headers.remove(h); }, diff --git a/ethcore/src/trace/db.rs b/ethcore/src/trace/db.rs index 3b3384144c7..4a849fbca80 100644 --- a/ethcore/src/trace/db.rs +++ b/ethcore/src/trace/db.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::sync::Arc; use bloomchain::{Number, Config as BloomConfig}; use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup}; -use util::{H256, H264, Database, DBTransaction, RwLock}; +use util::{H256, H264, Database, DBTransaction, RwLock, HeapSizeOf}; use header::BlockNumber; use trace::{LocalizedTrace, Config, Switch, Filter, Database as TraceDatabase, ImportRequest, DatabaseExtras, Error}; use db::{Key, Writable, Readable, CacheUpdatePolicy}; @@ -62,6 +62,12 @@ impl From for TraceGroupPosition { } } +impl HeapSizeOf for TraceGroupPosition { + fn heap_size_of_children(&self) -> usize { + 0 + } +} + /// Helper data structure created cause [u8; 6] does not implement Deref to &[u8]. pub struct TraceGroupKey([u8; 6]); @@ -155,7 +161,9 @@ impl TraceDB where T: DatabaseExtras { } fn cache_size(&self) -> usize { - 0 + let traces = self.traces.read().heap_size_of_children(); + let blooms = self.blooms.read().heap_size_of_children(); + traces + blooms } /// Let the cache system know that a cacheable item has been used. @@ -166,10 +174,11 @@ impl TraceDB where T: DatabaseExtras { /// Ticks our cache system and throws out any old data. pub fn collect_garbage(&self) { - let mut traces = self.traces.write(); - let mut blooms = self.blooms.write(); let mut cache_manager = self.cache_manager.write(); cache_manager.collect_carbage(|| self.cache_size(), | ids | { + let mut traces = self.traces.write(); + let mut blooms = self.blooms.write(); + for id in &ids { match *id { CacheID::Trace(ref h) => { traces.remove(h); }, diff --git a/ethcore/src/types/trace_types/flat.rs b/ethcore/src/types/trace_types/flat.rs index ad1e9bacea0..1000d918300 100644 --- a/ethcore/src/types/trace_types/flat.rs +++ b/ethcore/src/types/trace_types/flat.rs @@ -20,6 +20,7 @@ use std::collections::VecDeque; use std::mem; use ipc::binary::BinaryConvertError; use util::rlp::*; +use util::HeapSizeOf; use basic_types::LogBloom; use super::trace::{Action, Res}; @@ -47,6 +48,12 @@ impl FlatTrace { } } +impl HeapSizeOf for FlatTrace { + fn heap_size_of_children(&self) -> usize { + self.trace_address.heap_size_of_children() + } +} + impl Encodable for FlatTrace { fn rlp_append(&self, s: &mut RlpStream) { s.begin_list(4); @@ -82,6 +89,12 @@ impl From> for FlatTransactionTraces { } } +impl HeapSizeOf for FlatTransactionTraces { + fn heap_size_of_children(&self) -> usize { + self.0.heap_size_of_children() + } +} + impl FlatTransactionTraces { /// Returns bloom of all traces in the collection. pub fn bloom(&self) -> LogBloom { @@ -111,6 +124,12 @@ impl Into> for FlatTransactionTraces { #[derive(Debug, PartialEq, Clone)] pub struct FlatBlockTraces(Vec); +impl HeapSizeOf for FlatBlockTraces { + fn heap_size_of_children(&self) -> usize { + self.0.heap_size_of_children() + } +} + impl From> for FlatBlockTraces { fn from(v: Vec) -> Self { FlatBlockTraces(v) From 0674b680ec7845a3f34184ea46e145ea68c5bfdd Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 29 Jul 2016 13:49:45 +0200 Subject: [PATCH 5/6] trace cache config --- ethcore/src/trace/config.rs | 4 ++-- parity/cache.rs | 10 ++++++++++ parity/helpers.rs | 4 ++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ethcore/src/trace/config.rs b/ethcore/src/trace/config.rs index 9a9db9cf9a3..ff96cea74cf 100644 --- a/ethcore/src/trace/config.rs +++ b/ethcore/src/trace/config.rs @@ -82,8 +82,8 @@ impl Default for Config { levels: 3, elements_per_index: 16, }, - pref_cache_size: 3 * 1024 * 1024, - max_cache_size: 5 * 1024 * 1024, + pref_cache_size: 15 * 1024 * 1024, + max_cache_size: 20 * 1024 * 1024, } } } diff --git a/parity/cache.rs b/parity/cache.rs index 45f1cb5f5ca..12fe3f472e0 100644 --- a/parity/cache.rs +++ b/parity/cache.rs @@ -20,6 +20,7 @@ const MIN_BC_CACHE_MB: u32 = 4; const MIN_DB_CACHE_MB: u32 = 2; const MIN_BLOCK_QUEUE_SIZE_LIMIT_MB: u32 = 16; const DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB: u32 = 50; +const DEFAULT_TRACE_CACHE_SIZE: u32 = 20; /// Configuration for application cache sizes. /// All values are represented in MB. @@ -34,6 +35,8 @@ pub struct CacheConfig { blockchain: u32, /// Size of transaction queue cache. queue: u32, + /// Size of traces cache. + traces: u32, } impl Default for CacheConfig { @@ -49,6 +52,7 @@ impl CacheConfig { db: total * 7 / 8, blockchain: total / 8, queue: DEFAULT_BLOCK_QUEUE_SIZE_LIMIT_MB, + traces: DEFAULT_TRACE_CACHE_SIZE, } } @@ -58,6 +62,7 @@ impl CacheConfig { db: db, blockchain: blockchain, queue: queue, + traces: DEFAULT_TRACE_CACHE_SIZE, } } @@ -80,6 +85,11 @@ impl CacheConfig { pub fn blockchain(&self) -> u32 { max(self.blockchain, MIN_BC_CACHE_MB) } + + /// Size of the traces cache. + pub fn traces(&self) -> u32 { + self.traces + } } #[cfg(test)] diff --git a/parity/helpers.rs b/parity/helpers.rs index 881dd9c8f10..2705dd5eaf5 100644 --- a/parity/helpers.rs +++ b/parity/helpers.rs @@ -210,6 +210,10 @@ pub fn to_client_config( client_config.db_cache_size = Some(cache_config.db_state_cache_size() as usize); // db queue cache size, in bytes client_config.queue.max_mem_use = cache_config.queue() as usize * mb; + // in bytes + client_config.tracing.max_cache_size = cache_config.traces() as usize * mb; + // in bytes + client_config.tracing.pref_cache_size = cache_config.traces() as usize * 3 / 4 * mb; client_config.mode = mode; client_config.tracing.enabled = tracing; From a8cc5371c72c2ac94b897016ad46cc166b92b5b4 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 30 Jul 2016 13:29:06 +0200 Subject: [PATCH 6/6] fixed carbage typo --- ethcore/src/blockchain/blockchain.rs | 2 +- ethcore/src/cache_manager.rs | 2 +- ethcore/src/trace/db.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 3d9d26abae1..bd8488a66b0 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -863,7 +863,7 @@ impl BlockChain { /// Ticks our cache system and throws out any old data. pub fn collect_garbage(&self) { let mut cache_man = self.cache_man.write(); - cache_man.collect_carbage(|| self.cache_size().total(), | ids | { + cache_man.collect_garbage(|| self.cache_size().total(), | ids | { let mut block_headers = self.block_headers.write(); let mut block_bodies = self.block_bodies.write(); let mut block_details = self.block_details.write(); diff --git a/ethcore/src/cache_manager.rs b/ethcore/src/cache_manager.rs index b132d78aa80..f68e3c61699 100644 --- a/ethcore/src/cache_manager.rs +++ b/ethcore/src/cache_manager.rs @@ -45,7 +45,7 @@ impl CacheManager where T: Eq + Hash { } } - pub fn collect_carbage(&mut self, current_size: C, mut notify_unused: F) where C: Fn() -> usize, F: FnMut(HashSet) { + pub fn collect_garbage(&mut self, current_size: C, mut notify_unused: F) where C: Fn() -> usize, F: FnMut(HashSet) { if current_size() < self.pref_cache_size { self.rotate_cache_if_needed(); return; diff --git a/ethcore/src/trace/db.rs b/ethcore/src/trace/db.rs index 4a849fbca80..722d872736f 100644 --- a/ethcore/src/trace/db.rs +++ b/ethcore/src/trace/db.rs @@ -175,7 +175,7 @@ impl TraceDB where T: DatabaseExtras { /// Ticks our cache system and throws out any old data. pub fn collect_garbage(&self) { let mut cache_manager = self.cache_manager.write(); - cache_manager.collect_carbage(|| self.cache_size(), | ids | { + cache_manager.collect_garbage(|| self.cache_size(), | ids | { let mut traces = self.traces.write(); let mut blooms = self.blooms.write();