From 6d7104bc8a9d4b0dde4cb54d42c1a057e0505f0d Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 30 Oct 2023 11:29:38 +0800 Subject: [PATCH] refactor(core/fuzz): Fix some bugs inside fuzzer Signed-off-by: Xuanwo --- Cargo.lock | 1 + core/fuzz/Cargo.toml | 4 +++ core/fuzz/fuzz_reader.rs | 69 +++++++++++++++++++++++++++++++--------- core/fuzz/utils.rs | 8 ++++- 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 719810b15cdd..e5fa7d138ab9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4279,6 +4279,7 @@ dependencies = [ "rand 0.8.5", "sha2", "tokio", + "tracing-subscriber", "uuid", ] diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml index 8020dfaf49c3..036433934ba1 100644 --- a/core/fuzz/Cargo.toml +++ b/core/fuzz/Cargo.toml @@ -35,6 +35,10 @@ rand = "0.8" sha2 = { version = "0.10.6" } tokio = { version = "1", features = ["full"] } uuid = { version = "1", features = ["v4"] } +tracing-subscriber = { version = "0.3", features = [ + "env-filter", + "tracing-log", +] } [[bin]] name = "fuzz_reader" diff --git a/core/fuzz/fuzz_reader.rs b/core/fuzz/fuzz_reader.rs index d6eb19c1c0f0..799d6f52bc78 100644 --- a/core/fuzz/fuzz_reader.rs +++ b/core/fuzz/fuzz_reader.rs @@ -17,6 +17,7 @@ #![no_main] +use std::fmt::{Debug, Formatter}; use std::io::SeekFrom; use bytes::Bytes; @@ -35,20 +36,37 @@ mod utils; const MAX_DATA_SIZE: usize = 16 * 1024 * 1024; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] enum ReadAction { Read { size: usize }, Seek(SeekFrom), Next, } -#[derive(Debug, Clone)] +#[derive(Clone)] struct FuzzInput { + path: String, size: usize, range: BytesRange, actions: Vec, } +impl Debug for FuzzInput { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut actions = self.actions.clone(); + // Remove all Read(0) entry. + let empty = ReadAction::Read { size: 0 }; + actions.retain(|e| e != &empty); + + f.debug_struct("FuzzInput") + .field("path", &self.path) + .field("size", &self.size) + .field("range", &self.range.to_string()) + .field("actions", &actions) + .finish() + } +} + impl Arbitrary<'_> for FuzzInput { fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { let total_size = u.int_in_range(1..=MAX_DATA_SIZE)?; @@ -109,6 +127,7 @@ impl Arbitrary<'_> for FuzzInput { } Ok(FuzzInput { + path: uuid::Uuid::new_v4().to_string(), size: total_size, range, actions, @@ -142,17 +161,30 @@ impl ReadChecker { } } - fn check_read(&mut self, n: usize, output: &[u8]) { - if n == 0 { + fn check_read(&mut self, buf_size: usize, output: &[u8]) { + if buf_size == 0 { assert_eq!( output.len(), 0, - "check read failed: output bs is not empty when read size is 0" + "check read failed: output must be empty if buf_size is 0" + ); + return; + } + + if buf_size > 0 && output.is_empty() { + assert!( + self.cur >= self.ranged_data.len(), + "check read failed: no data read means cur must outsides of ranged_data", ); return; } - let expected = &self.ranged_data[self.cur..self.cur + n]; + assert!( + self.cur + output.len() <= self.ranged_data.len(), + "check read failed: cur + output length must be less than ranged_data length, cur: {}, output: {}, ranged_data: {}", self.cur, output.len(), self.ranged_data.len(), + ); + + let expected = &self.ranged_data[self.cur..self.cur + output.len()]; // Check the read result assert_eq!( @@ -162,7 +194,7 @@ impl ReadChecker { ); // Update the current position - self.cur += n; + self.cur += output.len(); } fn check_seek(&mut self, seek_from: SeekFrom, output: Result) { @@ -206,7 +238,8 @@ impl ReadChecker { "{:x}", Sha256::digest(&self.ranged_data[self.cur..self.cur + output.len()]) ), - "check next failed: output bs is different with expected bs", + "check next failed: output bs is different with expected bs, current: {}, output length: {}", + self.cur, output.len(), ); // update the current position @@ -221,19 +254,20 @@ impl ReadChecker { } async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let mut checker = ReadChecker::new(input.size, input.range); - op.write(&path, checker.raw_data.clone()).await?; + op.write(&input.path, checker.raw_data.clone()).await?; - let mut o = op.reader_with(&path).range(input.range.to_range()).await?; + let mut o = op + .reader_with(&input.path) + .range(input.range.to_range()) + .await?; for action in input.actions { match action { ReadAction::Read { size } => { let mut buf = vec![0; size]; let n = o.read(&mut buf).await?; - checker.check_read(n, &buf[..n]); + checker.check_read(size, &buf[..n]); } ReadAction::Seek(seek_from) => { @@ -248,12 +282,17 @@ async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> { } } - op.delete(&path).await?; + op.delete(&input.path).await?; Ok(()) } fuzz_target!(|input: FuzzInput| { let _ = dotenvy::dotenv(); + let _ = tracing_subscriber::fmt() + .pretty() + .with_test_writer() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init(); let runtime = tokio::runtime::Runtime::new().expect("init runtime must succeed"); @@ -261,7 +300,7 @@ fuzz_target!(|input: FuzzInput| { runtime.block_on(async { fuzz_reader(op, input.clone()) .await - .unwrap_or_else(|_| panic!("fuzz reader must succeed")); + .unwrap_or_else(|err| panic!("fuzz reader must succeed: {err:?}")); }) } }); diff --git a/core/fuzz/utils.rs b/core/fuzz/utils.rs index 37598fdb3713..e83e4c18473c 100644 --- a/core/fuzz/utils.rs +++ b/core/fuzz/utils.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use opendal::layers::{LoggingLayer, RetryLayer}; use std::env; use std::str::FromStr; @@ -44,5 +45,10 @@ pub fn init_service() -> Option { }) .collect(); - Some(Operator::via_map(scheme, envs).unwrap_or_else(|_| panic!("init {} must succeed", scheme))) + Some( + Operator::via_map(scheme, envs) + .unwrap_or_else(|_| panic!("init {} must succeed", scheme)) + .layer(LoggingLayer::default()) + .layer(RetryLayer::default()), + ) }