From 97685318fcbc4943bbe26540a673c46308c95850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Thu, 18 Jan 2018 19:37:05 +0000 Subject: [PATCH 1/5] kvdb-rocksdb: update rust-rocksdb version --- Cargo.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a0069ec1bf..f87922f96e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2689,7 +2689,7 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/paritytech/rust-rocksdb#7adec2311d31387a832b0ef051472cdef906b480" +source = "git+https://github.com/paritytech/rust-rocksdb#ecf06adf3148ab10f6f7686b724498382ff4f36e" dependencies = [ "libc 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", "local-encoding 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2699,10 +2699,11 @@ dependencies = [ [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/paritytech/rust-rocksdb#7adec2311d31387a832b0ef051472cdef906b480" +source = "git+https://github.com/paritytech/rust-rocksdb#ecf06adf3148ab10f6f7686b724498382ff4f36e" dependencies = [ "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", + "local-encoding 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "snappy-sys 0.1.0 (git+https://github.com/paritytech/rust-snappy)", ] From 6c1acee4f001c03778d094ab8f2b82be1d5b2592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Thu, 18 Jan 2018 21:50:05 +0000 Subject: [PATCH 2/5] kvdb-rocksdb: mark corruptions and attempt repair on db open --- util/kvdb-rocksdb/src/lib.rs | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index df605855c76..878f2cb90a5 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -32,7 +32,7 @@ use std::cmp; use std::collections::HashMap; use std::marker::PhantomData; use std::path::{PathBuf, Path}; -use std::{mem, fs, io}; +use std::{fs, io, mem, result}; use parking_lot::{Mutex, MutexGuard, RwLock}; use rocksdb::{ @@ -257,6 +257,18 @@ pub struct Database { flushing_lock: Mutex, } +#[inline] +fn mark_corruption>(path: P, res: result::Result) -> result::Result { + if let Err(ref s) = res { + if s.starts_with("Corruption:") { + warn!("DB corruption detected: {}. Repair will be triggered on next restart", s); + let _ = fs::File::create(path.as_ref().join("CORRUPTED")); + } + } + + res +} + impl Database { /// Open database with default settings. pub fn open_default(path: &str) -> Result { @@ -287,6 +299,14 @@ impl Database { block_opts.set_cache(cache); } + // attempt database repair if it has been previously marked as corrupted + let db_corrupted = Path::new(path).join("CORRUPTED"); + if db_corrupted.exists() { + warn!("DB {} has been previously marked as corrupted, attempting repair.", path); + DB::repair(&opts, path)?; + fs::remove_file(db_corrupted)?; + } + let columns = config.columns.unwrap_or(0) as usize; let mut cf_options = Vec::with_capacity(columns); @@ -425,7 +445,11 @@ impl Database { } } } - db.write_opt(batch, &self.write_opts)?; + + mark_corruption( + &self.path, + db.write_opt(batch, &self.write_opts))?; + for column in self.flushing.write().iter_mut() { column.clear(); column.shrink_to_fit(); @@ -471,7 +495,10 @@ impl Database { }, } } - db.write_opt(batch, &self.write_opts).map_err(Into::into) + + mark_corruption( + &self.path, + db.write_opt(batch, &self.write_opts)).map_err(Into::into) }, None => Err("Database is closed".into()) } From 79e1d02d815ec9dd67521694884e2b3ea6213d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Fri, 19 Jan 2018 00:21:11 +0000 Subject: [PATCH 3/5] kvdb-rocksdb: better corruption detection on open --- util/kvdb-rocksdb/src/lib.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index 878f2cb90a5..e8fab5d1f0e 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -261,7 +261,7 @@ pub struct Database { fn mark_corruption>(path: P, res: result::Result) -> result::Result { if let Err(ref s) = res { if s.starts_with("Corruption:") { - warn!("DB corruption detected: {}. Repair will be triggered on next restart", s); + warn!("DB corrupted: {}. Repair will be triggered on next restart", s); let _ = fs::File::create(path.as_ref().join("CORRUPTED")); } } @@ -269,6 +269,10 @@ fn mark_corruption>(path: P, res: result::Result) - res } +fn is_corrupted(s: &str) -> bool { + s.starts_with("Corruption:") || s.starts_with("Invalid argument: You have to open all column families") +} + impl Database { /// Open database with default settings. pub fn open_default(path: &str) -> Result { @@ -302,7 +306,7 @@ impl Database { // attempt database repair if it has been previously marked as corrupted let db_corrupted = Path::new(path).join("CORRUPTED"); if db_corrupted.exists() { - warn!("DB {} has been previously marked as corrupted, attempting repair.", path); + warn!("DB has been previously marked as corrupted, attempting repair"); DB::repair(&opts, path)?; fs::remove_file(db_corrupted)?; } @@ -326,12 +330,11 @@ impl Database { let mut cfs: Vec = Vec::new(); let db = match config.columns { - Some(columns) => { + Some(_) => { match DB::open_cf(&opts, path, &cfnames, &cf_options) { Ok(db) => { cfs = cfnames.iter().map(|n| db.cf_handle(n) .expect("rocksdb opens a cf_handle for each cfname; qed")).collect(); - assert!(cfs.len() == columns as usize); Ok(db) } Err(_) => { @@ -341,7 +344,7 @@ impl Database { cfs = cfnames.iter().enumerate().map(|(i, n)| db.create_cf(n, &cf_options[i])).collect::<::std::result::Result<_, _>>()?; Ok(db) }, - err @ Err(_) => err, + err => err, } } } @@ -351,14 +354,18 @@ impl Database { let db = match db { Ok(db) => db, - Err(ref s) if s.starts_with("Corruption:") => { - info!("{}", s); - info!("Attempting DB repair for {}", path); + Err(ref s) if is_corrupted(s) => { + warn!("DB corrupted: {}, attempting repair", s); DB::repair(&opts, path)?; match cfnames.is_empty() { true => DB::open(&opts, path)?, - false => DB::open_cf(&opts, path, &cfnames, &cf_options)? + false => { + let db = DB::open_cf(&opts, path, &cfnames, &cf_options)?; + cfs = cfnames.iter().map(|n| db.cf_handle(n) + .expect("rocksdb opens a cf_handle for each cfname; qed")).collect(); + db + }, } }, Err(s) => { return Err(s.into()); } From 740d466eafd89f72d29211661e3e7ff2dfc872d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Fri, 19 Jan 2018 02:15:17 +0000 Subject: [PATCH 4/5] kvdb-rocksdb: add corruption_file_name const --- util/kvdb-rocksdb/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index e8fab5d1f0e..e538fbaec9d 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -262,7 +262,7 @@ fn mark_corruption>(path: P, res: result::Result) - if let Err(ref s) = res { if s.starts_with("Corruption:") { warn!("DB corrupted: {}. Repair will be triggered on next restart", s); - let _ = fs::File::create(path.as_ref().join("CORRUPTED")); + let _ = fs::File::create(path.as_ref().join(Database::CORRUPTION_FILE_NAME)); } } @@ -274,6 +274,8 @@ fn is_corrupted(s: &str) -> bool { } impl Database { + const CORRUPTION_FILE_NAME: &'static str = "CORRUPTED"; + /// Open database with default settings. pub fn open_default(path: &str) -> Result { Database::open(&DatabaseConfig::default(), path) @@ -304,7 +306,7 @@ impl Database { } // attempt database repair if it has been previously marked as corrupted - let db_corrupted = Path::new(path).join("CORRUPTED"); + let db_corrupted = Path::new(path).join(Database::CORRUPTION_FILE_NAME); if db_corrupted.exists() { warn!("DB has been previously marked as corrupted, attempting repair"); DB::repair(&opts, path)?; From 6c0335037d294b941734f3cb72c67a5e5a0adeaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Fri, 19 Jan 2018 12:34:59 +0000 Subject: [PATCH 5/5] kvdb-rocksdb: rename mark_corruption to check_for_corruption --- util/kvdb-rocksdb/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index e538fbaec9d..91cee86bdd2 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -258,7 +258,7 @@ pub struct Database { } #[inline] -fn mark_corruption>(path: P, res: result::Result) -> result::Result { +fn check_for_corruption>(path: P, res: result::Result) -> result::Result { if let Err(ref s) = res { if s.starts_with("Corruption:") { warn!("DB corrupted: {}. Repair will be triggered on next restart", s); @@ -455,7 +455,7 @@ impl Database { } } - mark_corruption( + check_for_corruption( &self.path, db.write_opt(batch, &self.write_opts))?; @@ -505,7 +505,7 @@ impl Database { } } - mark_corruption( + check_for_corruption( &self.path, db.write_opt(batch, &self.write_opts)).map_err(Into::into) },