Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fuzz] Remove more fuzz targets #4737

Merged
merged 2 commits into from
Aug 19, 2022
+94 −273
Merged

[fuzz] Remove more fuzz targets #4737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
45bd5cf
[fuzz] Remove the `differential` fuzz target
abrown Aug 19, 2022
9c96b88
[fuzz] Rename `differential_meta` to `differential`
abrown Aug 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 0 additions & 130 deletions crates/fuzzing/src/oracles.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,136 +380,6 @@ pub fn differential(
Ok(())
}

/// Instantiate the given Wasm module with each `Config` and call all of its
/// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are
/// or aren't enabled for different configs, we should get the same results when
/// we call the exported functions for all of our different configs.
///
/// Returns `None` if a fuzz configuration was rejected (should happen rarely).
pub fn differential_execution(
wasm: &[u8],
module_config: &generators::ModuleConfig,
configs: &[generators::WasmtimeConfig],
) -> Option<()> {
use std::collections::{HashMap, HashSet};

// We need at least two configs.
if configs.len() < 2
// And all the configs should be unique.
|| configs.iter().collect::<HashSet<_>>().len() != configs.len()
{
return None;
}

let mut export_func_results: HashMap<String, Result<Box<[Val]>, Trap>> = Default::default();
log_wasm(&wasm);

for fuzz_config in configs {
let fuzz_config = generators::Config {
module_config: module_config.clone(),
wasmtime: fuzz_config.clone(),
};
log::debug!("fuzz config: {:?}", fuzz_config);

let mut store = fuzz_config.to_store();
let module = compile_module(store.engine(), &wasm, true, &fuzz_config)?;

// TODO: we should implement tracing versions of these dummy imports
// that record a trace of the order that imported functions were called
// in and with what values. Like the results of exported functions,
// calls to imports should also yield the same values for each
// configuration, and we should assert that.
let instance = match instantiate_with_dummy(&mut store, &module) {
Some(instance) => instance,
None => continue,
};

let exports = instance
.exports(&mut store)
.filter_map(|e| {
let name = e.name().to_string();
e.into_func().map(|f| (name, f))
})
.collect::<Vec<_>>();
for (name, f) in exports {
log::debug!("invoke export {:?}", name);
let ty = f.ty(&store);
let params = dummy::dummy_values(ty.params());
let mut results = vec![Val::I32(0); ty.results().len()];
let this_result = f
.call(&mut store, &params, &mut results)
.map(|()| results.into())
.map_err(|e| e.downcast::<Trap>().unwrap());

let existing_result = export_func_results
.entry(name.to_string())
.or_insert_with(|| this_result.clone());
assert_same_export_func_result(&existing_result, &this_result, &name);
}
}

return Some(());

fn assert_same_export_func_result(
lhs: &Result<Box<[Val]>, Trap>,
rhs: &Result<Box<[Val]>, Trap>,
func_name: &str,
) {
let fail = || {
panic!(
"differential fuzzing failed: exported func {} returned two \
different results: {:?} != {:?}",
func_name, lhs, rhs
)
};

match (lhs, rhs) {
// Different compilation settings can lead to different amounts
// of stack space being consumed, so if either the lhs or the rhs
// hit a stack overflow then we discard the result of the other side
// since if it ran successfully or trapped that's ok in both
// situations.
(Err(e), _) | (_, Err(e)) if e.trap_code() == Some(TrapCode::StackOverflow) => {}

(Err(a), Err(b)) => {
if a.trap_code() != b.trap_code() {
fail();
}
}
(Ok(lhs), Ok(rhs)) => {
if lhs.len() != rhs.len() {
fail();
}
for (lhs, rhs) in lhs.iter().zip(rhs.iter()) {
match (lhs, rhs) {
(Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue,
(Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue,
(Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue,
(Val::F32(lhs), Val::F32(rhs)) if f32_equal(*lhs, *rhs) => continue,
(Val::F64(lhs), Val::F64(rhs)) if f64_equal(*lhs, *rhs) => continue,
(Val::ExternRef(_), Val::ExternRef(_))
| (Val::FuncRef(_), Val::FuncRef(_)) => continue,
_ => fail(),
}
}
}
_ => fail(),
}
}
}

fn f32_equal(a: u32, b: u32) -> bool {
let a = f32::from_bits(a);
let b = f32::from_bits(b);
a == b || (a.is_nan() && b.is_nan())
}

fn f64_equal(a: u64, b: u64) -> bool {
let a = f64::from_bits(a);
let b = f64::from_bits(b);
a == b || (a.is_nan() && b.is_nan())
}

/// Invoke the given API calls.
pub fn make_api_calls(api: generators::api::ApiCalls) {
use crate::generators::api::ApiCall;
Expand Down
6 changes: 0 additions & 6 deletions fuzz/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,6 @@ path = "fuzz_targets/differential.rs"
test = false
doc = false

[[bin]]
name = "differential_meta"
path = "fuzz_targets/differential_meta.rs"
test = false
doc = false

[[bin]]
name = "differential_v8"
path = "fuzz_targets/differential_v8.rs"
Expand Down
4 changes: 1 addition & 3 deletions fuzz/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,7 @@ At the time of writing, we have the following fuzz targets:
to its source, yielding a function A', and checks that A compiled +
incremental compilation generates the same machine code as if A' was compiled
from scratch.
* `differential`: Generate a Wasm module and check that Wasmtime returns
the same results when run with two different configurations.
* `differential_meta`: Generate a Wasm module, evaluate each exported function
* `differential`: Generate a Wasm module, evaluate each exported function
with random inputs, and check that Wasmtime returns the same results as a
choice of another engine: the Wasm spec interpreter (see the
`wasm-spec-interpreter` crate), the `wasmi` interpreter, or Wasmtime itself
Expand Down
118 changes: 93 additions & 25 deletions fuzz/fuzz_targets/differential.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,108 @@

use libfuzzer_sys::arbitrary::{Result, Unstructured};
use libfuzzer_sys::fuzz_target;
use wasmtime_fuzzing::generators::InstanceAllocationStrategy;
use wasmtime_fuzzing::{generators, oracles};
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering::SeqCst;
use std::sync::Once;
use wasmtime_fuzzing::generators::{Config, DiffValue, SingleInstModule};
use wasmtime_fuzzing::oracles::diff_spec;
use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance;
use wasmtime_fuzzing::oracles::{differential, engine, log_wasm};

// Upper limit on the number of invocations for each WebAssembly function
// executed by this fuzz target.
const NUM_INVOCATIONS: usize = 5;

// Keep track of how many WebAssembly modules we actually executed (i.e. ran to
// completion) versus how many were tried.
static TOTAL_INVOCATIONS: AtomicUsize = AtomicUsize::new(0);
static TOTAL_SUCCESSES: AtomicUsize = AtomicUsize::new(0);
static TOTAL_ATTEMPTED: AtomicUsize = AtomicUsize::new(0);

// The spec interpreter requires special one-time setup.
static SETUP: Once = Once::new();

fuzz_target!(|data: &[u8]| {
// errors in `run` have to do with not enough input in `data`, which we
// To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on
// `setup_ocaml_runtime`.
SETUP.call_once(|| diff_spec::setup_ocaml_runtime());

// Errors in `run` have to do with not enough input in `data`, which we
// ignore here since it doesn't affect how we'd like to fuzz.
drop(run(data));
drop(run(&data));
});

fn run(data: &[u8]) -> Result<()> {
let successes = TOTAL_SUCCESSES.load(SeqCst);
let attempts = TOTAL_ATTEMPTED.fetch_add(1, SeqCst);
if attempts > 1 && attempts % 1_000 == 0 {
println!("=== Execution rate ({} successes / {} attempted modules): {}% (total invocations: {}) ===",
successes,
attempts,
successes as f64 / attempts as f64 * 100f64,
TOTAL_INVOCATIONS.load(SeqCst)
);
}

let mut u = Unstructured::new(data);
let mut config: Config = u.arbitrary()?;
config.set_differential_config();

// Generate the Wasm module.
let wasm = if u.arbitrary()? {
// TODO figure out if this always eats up the rest of the unstructured;
// can we limit the number of instructions/functions.
let module = config.generate(&mut u, Some(1000))?;
module.to_bytes()
} else {
let module = SingleInstModule::new(&mut u, &mut config.module_config)?;
module.to_bytes()
};
log_wasm(&wasm);

// Choose a left-hand side Wasm engine.
let lhs = engine::choose(&mut u, &config)?;
let lhs_instance = lhs.instantiate(&wasm);

// Choose a right-hand side Wasm engine--this will always be Wasmtime.
let rhs_store = config.to_store();
let rhs_module = wasmtime::Module::new(rhs_store.engine(), &wasm).unwrap();
let rhs_instance = WasmtimeInstance::new(rhs_store, rhs_module);

// If we fail to instantiate, check that both sides do.
let (mut lhs_instance, mut rhs_instance) = match (lhs_instance, rhs_instance) {
(Ok(l), Ok(r)) => (l, r),
(Err(_), Err(_)) => return Ok(()), // TODO match the error messages.
(l, r) => panic!(
"failed to instantiate only one side: {:?} != {:?}",
l.err(),
r.err()
),
};

// Call each exported function with different sets of arguments.
for (name, signature) in rhs_instance.exported_functions() {
let mut invocations = 0;
loop {
let arguments = signature
.params()
.map(|t| DiffValue::arbitrary_of_type(&mut u, t.try_into().unwrap()))
.collect::<Result<Vec<_>>>()?;
differential(lhs_instance.as_mut(), &mut rhs_instance, &name, &arguments)
.expect("failed to run differential evaluation");

let mut config: generators::Config = u.arbitrary()?;
let module = config.generate(&mut u, Some(1000))?;

let lhs = config.wasmtime;
let mut rhs: generators::WasmtimeConfig = u.arbitrary()?;

// Use the same allocation strategy between the two configs.
//
// Ideally this wouldn't be necessary, but if the lhs is using ondemand
// and the rhs is using the pooling allocator (or vice versa), then
// the module may have been generated in such a way that is incompatible
// with the other allocation strategy.
//
// We can remove this in the future when it's possible to access the
// fields of `wasm_smith::Module` to constrain the pooling allocator
// based on what was actually generated.
rhs.strategy = lhs.strategy.clone();
if let InstanceAllocationStrategy::Pooling { .. } = &rhs.strategy {
// Also use the same memory configuration when using the pooling allocator
rhs.memory_config = lhs.memory_config.clone();
// We evaluate the same function with different arguments until we
// hit a predetermined limit or we run out of unstructured data--it
// does not make sense to re-evaluate the same arguments over and
// over.
invocations += 1;
TOTAL_INVOCATIONS.fetch_add(1, SeqCst);
if invocations > NUM_INVOCATIONS || u.is_empty() {
break;
}
}
}

oracles::differential_execution(&module.to_bytes(), &config.module_config, &[lhs, rhs]);
TOTAL_SUCCESSES.fetch_add(1, SeqCst);
Ok(())
}
Loading