Skip to content

Commit

Permalink
Virtual file support (#701)
Browse files Browse the repository at this point in the history 
* Add support for virtual files (eg, not backed by an OS file).

Virtual files are implemented through trait objects, with a default
implementation that tries to behave like on-disk files, but entirely
backed by in-memory structures.

Co-authored-by: Dan Gohman <[email protected]>
  • Loading branch information
@iximeow @sunfishcode
iximeow and sunfishcode authored Mar 6, 2020
1 parent 7f7196a commit 7e0d9de
Show file tree
Hide file tree
Showing 19 changed files with 1,567 additions and 187 deletions.
111 changes: 92 additions & 19 deletions crates/test-programs/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,19 @@ fn main() {
#[cfg(feature = "test_programs")]
mod wasi_tests {
use std::env;
use std::fs::{read_dir, DirEntry, File};
use std::fs::{read_dir, File};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};

#[derive(Clone, Copy, Debug)]
enum PreopenType {
/// Preopens should be satisfied with real OS files.
OS,
/// Preopens should be satisfied with virtual files.
Virtual,
}

pub(super) fn build_and_generate_tests() {
// Validate if any of test sources are present and if they changed
// This should always work since there is no submodule to init anymore
Expand Down Expand Up @@ -103,34 +111,52 @@ mod wasi_tests {
.replace("-", "_")
)?;
writeln!(out, " use super::{{runtime, utils, setup_log}};")?;
writeln!(out, " use runtime::PreopenType;")?;
for dir_entry in dir_entries {
write_testsuite_tests(out, dir_entry, testsuite)?;
let test_path = dir_entry.path();
let stemstr = test_path
.file_stem()
.expect("file_stem")
.to_str()
.expect("to_str");

if no_preopens(testsuite, stemstr) {
write_testsuite_tests(out, &test_path, testsuite, PreopenType::OS)?;
} else {
write_testsuite_tests(out, &test_path, testsuite, PreopenType::OS)?;
write_testsuite_tests(out, &test_path, testsuite, PreopenType::Virtual)?;
}
}
writeln!(out, "}}")?;
Ok(())
}

fn write_testsuite_tests(
out: &mut File,
dir_entry: DirEntry,
path: &Path,
testsuite: &str,
preopen_type: PreopenType,
) -> io::Result<()> {
let path = dir_entry.path();
let stemstr = path
.file_stem()
.expect("file_stem")
.to_str()
.expect("to_str");

writeln!(out, " #[test]")?;
if ignore(testsuite, stemstr) {
let test_fn_name = format!(
"{}{}",
&stemstr.replace("-", "_"),
if let PreopenType::Virtual = preopen_type {
"_virtualfs"
} else {
""
}
);
if ignore(testsuite, &test_fn_name) {
writeln!(out, " #[ignore]")?;
}
writeln!(
out,
" fn r#{}() -> anyhow::Result<()> {{",
&stemstr.replace("-", "_")
)?;
writeln!(out, " fn r#{}() -> anyhow::Result<()> {{", test_fn_name,)?;
writeln!(out, " setup_log();")?;
writeln!(
out,
Expand All @@ -145,16 +171,25 @@ mod wasi_tests {
let workspace = if no_preopens(testsuite, stemstr) {
"None"
} else {
writeln!(
out,
" let workspace = utils::prepare_workspace(&bin_name)?;"
)?;
"Some(workspace.path())"
match preopen_type {
PreopenType::OS => {
writeln!(
out,
" let workspace = utils::prepare_workspace(&bin_name)?;"
)?;
"Some(workspace.path())"
}
PreopenType::Virtual => "Some(std::path::Path::new(&bin_name))",
}
};
writeln!(
out,
" runtime::instantiate(&data, &bin_name, {})",
workspace
" runtime::instantiate(&data, &bin_name, {}, {})",
workspace,
match preopen_type {
PreopenType::OS => "PreopenType::OS",
PreopenType::Virtual => "PreopenType::Virtual",
}
)?;
writeln!(out, " }}")?;
writeln!(out)?;
Expand All @@ -164,8 +199,30 @@ mod wasi_tests {
cfg_if::cfg_if! {
if #[cfg(not(windows))] {
/// Ignore tests that aren't supported yet.
fn ignore(_testsuite: &str, _name: &str) -> bool {
false
fn ignore(testsuite: &str, name: &str) -> bool {
if testsuite == "wasi-tests" {
match name {
// TODO: virtfs files cannot be poll_oneoff'd yet
"poll_oneoff_virtualfs" => true,
// TODO: virtfs does not support filetimes yet.
"path_filestat_virtualfs" |
"fd_filestat_set_virtualfs" => true,
// TODO: virtfs does not support symlinks yet.
"nofollow_errors_virtualfs" |
"path_link_virtualfs" |
"readlink_virtualfs" |
"readlink_no_buffer_virtualfs" |
"dangling_symlink_virtualfs" |
"symlink_loop_virtualfs" |
"path_symlink_trailing_slashes_virtualfs" => true,
// TODO: virtfs does not support rename yet.
"path_rename_trailing_slashes_virtualfs" |
"path_rename_virtualfs" => true,
_ => false,
}
} else {
unreachable!()
}
}
} else {
/// Ignore tests that aren't supported yet.
Expand All @@ -178,6 +235,22 @@ mod wasi_tests {
"truncation_rights" => true,
"path_link" => true,
"dangling_fd" => true,
// TODO: virtfs files cannot be poll_oneoff'd yet
"poll_oneoff_virtualfs" => true,
// TODO: virtfs does not support filetimes yet.
"path_filestat_virtualfs" |
"fd_filestat_set_virtualfs" => true,
// TODO: virtfs does not support symlinks yet.
"nofollow_errors_virtualfs" |
"path_link_virtualfs" |
"readlink_virtualfs" |
"readlink_no_buffer_virtualfs" |
"dangling_symlink_virtualfs" |
"symlink_loop_virtualfs" |
"path_symlink_trailing_slashes_virtualfs" => true,
// TODO: virtfs does not support rename yet.
"path_rename_trailing_slashes_virtualfs" |
"path_rename_virtualfs" => true,
_ => false,
}
} else {
Expand Down
44 changes: 29 additions & 15 deletions crates/test-programs/tests/wasm_tests/runtime.rs
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,44 @@
use anyhow::{bail, Context};
use std::fs::File;
use std::path::Path;
use wasi_common::VirtualDirEntry;
use wasmtime::{Instance, Module, Store};

pub fn instantiate(data: &[u8], bin_name: &str, workspace: Option<&Path>) -> anyhow::Result<()> {
let store = Store::default();

let get_preopens = |workspace: Option<&Path>| -> anyhow::Result<Vec<_>> {
if let Some(workspace) = workspace {
let preopen_dir = wasi_common::preopen_dir(workspace)
.context(format!("error while preopening {:?}", workspace))?;
#[derive(Clone, Copy, Debug)]
pub enum PreopenType {
/// Preopens should be satisfied with real OS files.
OS,
/// Preopens should be satisfied with virtual files.
Virtual,
}

Ok(vec![(".".to_owned(), preopen_dir)])
} else {
Ok(vec![])
}
};
pub fn instantiate(
data: &[u8],
bin_name: &str,
workspace: Option<&Path>,
preopen_type: PreopenType,
) -> anyhow::Result<()> {
let store = Store::default();

// Create our wasi context with pretty standard arguments/inheritance/etc.
// Additionally register andy preopened directories if we have them.
// Additionally register any preopened directories if we have them.
let mut builder = wasi_common::WasiCtxBuilder::new();

builder.arg(bin_name).arg(".").inherit_stdio();

for (dir, file) in get_preopens(workspace)? {
builder.preopened_dir(file, dir);
if let Some(workspace) = workspace {
match preopen_type {
PreopenType::OS => {
let preopen_dir = wasi_common::preopen_dir(workspace)
.context(format!("error while preopening {:?}", workspace))?;
builder.preopened_dir(preopen_dir, ".");
}
PreopenType::Virtual => {
// we can ignore the workspace path for virtual preopens because virtual preopens
// don't exist in the filesystem anyway - no name conflict concerns.
builder.preopened_virt(VirtualDirEntry::empty_directory(), ".");
}
}
}

// The nonstandard thing we do with `WasiCtxBuilder` is to ensure that
Expand Down
6 changes: 3 additions & 3 deletions crates/test-programs/wasi-tests/src/bin/fd_flags_set.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
)
.expect("reading file"),
buffer.len(),
"shoudl read {} bytes",
"should read {} bytes",
buffer.len()
);

Expand Down Expand Up @@ -87,7 +87,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
)
.expect("reading file"),
buffer.len(),
"shoudl read {} bytes",
"should read {} bytes",
buffer.len()
);

Expand Down Expand Up @@ -126,7 +126,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
)
.expect("reading file"),
buffer.len(),
"shoudl read {} bytes",
"should read {} bytes",
buffer.len()
);

Expand Down
6 changes: 3 additions & 3 deletions crates/test-programs/wasi-tests/src/bin/remove_directory_trailing_slashes.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@ unsafe fn test_remove_directory_trailing_slashes(dir_fd: wasi::Fd) {

wasi::path_create_directory(dir_fd, "dir").expect("creating a directory");

// Test that removing it with a trailing flash succeeds.
// Test that removing it with a trailing slash succeeds.
wasi::path_remove_directory(dir_fd, "dir/")
.expect("remove_directory with a trailing slash on a directory should succeed");

// Create a temporary file.
create_file(dir_fd, "file");

// Test that removing it with no trailing flash fails.
// Test that removing it with no trailing slash fails.
assert_eq!(
wasi::path_remove_directory(dir_fd, "file")
.expect_err("remove_directory without a trailing slash on a file should fail")
Expand All @@ -27,7 +27,7 @@ unsafe fn test_remove_directory_trailing_slashes(dir_fd: wasi::Fd) {
"errno should be ERRNO_NOTDIR"
);

// Test that removing it with a trailing flash fails.
// Test that removing it with a trailing slash fails.
assert_eq!(
wasi::path_remove_directory(dir_fd, "file/")
.expect_err("remove_directory with a trailing slash on a file should fail")
Expand Down
62 changes: 56 additions & 6 deletions crates/wasi-common/src/ctx.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
use crate::fdentry::FdEntry;
use crate::fdentry::{Descriptor, FdEntry};
use crate::sys::fdentry_impl::OsHandle;
use crate::virtfs::{VirtualDir, VirtualDirEntry};
use crate::{wasi, Error, Result};
use std::borrow::Borrow;
use std::collections::HashMap;
Expand Down Expand Up @@ -61,7 +63,7 @@ impl PendingCString {
/// A builder allowing customizable construction of `WasiCtx` instances.
pub struct WasiCtxBuilder {
fds: Option<HashMap<wasi::__wasi_fd_t, PendingFdEntry>>,
preopens: Option<Vec<(PathBuf, File)>>,
preopens: Option<Vec<(PathBuf, Descriptor)>>,
args: Option<Vec<PendingCString>>,
env: Option<HashMap<PendingCString, PendingCString>>,
}
Expand Down Expand Up @@ -222,10 +224,46 @@ impl WasiCtxBuilder {

/// Add a preopened directory.
pub fn preopened_dir<P: AsRef<Path>>(&mut self, dir: File, guest_path: P) -> &mut Self {
self.preopens.as_mut().unwrap().push((
guest_path.as_ref().to_owned(),
Descriptor::OsHandle(OsHandle::from(dir)),
));
self
}

/// Add a preopened virtual directory.
pub fn preopened_virt<P: AsRef<Path>>(
&mut self,
dir: VirtualDirEntry,
guest_path: P,
) -> &mut Self {
fn populate_directory(virtentry: HashMap<String, VirtualDirEntry>, dir: &mut VirtualDir) {
for (path, entry) in virtentry.into_iter() {
match entry {
VirtualDirEntry::Directory(dir_entries) => {
let mut subdir = VirtualDir::new(true);
populate_directory(dir_entries, &mut subdir);
dir.add_dir(subdir, path);
}
VirtualDirEntry::File(content) => {
dir.add_file(content, path);
}
}
}
}

let dir = if let VirtualDirEntry::Directory(entries) = dir {
let mut dir = VirtualDir::new(true);
populate_directory(entries, &mut dir);
Box::new(dir)
} else {
panic!("the root of a VirtualDirEntry tree must be a VirtualDirEntry::Directory");
};

self.preopens
.as_mut()
.unwrap()
.push((guest_path.as_ref().to_owned(), dir));
.push((guest_path.as_ref().to_owned(), Descriptor::VirtualFile(dir)));
self
}

Expand Down Expand Up @@ -271,7 +309,7 @@ impl WasiCtxBuilder {
fds.insert(fd, f()?);
}
PendingFdEntry::File(f) => {
fds.insert(fd, FdEntry::from(f)?);
fds.insert(fd, FdEntry::from(Descriptor::OsHandle(OsHandle::from(f)))?);
}
}
}
Expand All @@ -284,8 +322,20 @@ impl WasiCtxBuilder {
// unnecessarily if we have exactly the maximum number of file descriptors.
preopen_fd = preopen_fd.checked_add(1).ok_or(Error::ENFILE)?;

if !dir.metadata()?.is_dir() {
return Err(Error::EBADF);
match &dir {
Descriptor::OsHandle(handle) => {
if !handle.metadata()?.is_dir() {
return Err(Error::EBADF);
}
}
Descriptor::VirtualFile(virt) => {
if virt.get_file_type() != wasi::__WASI_FILETYPE_DIRECTORY {
return Err(Error::EBADF);
}
}
Descriptor::Stdin | Descriptor::Stdout | Descriptor::Stderr => {
panic!("implementation error, stdin/stdout/stderr shouldn't be in the list of preopens");
}
}

// We don't currently allow setting file descriptors other than 0-2, but this will avoid
Expand Down
Loading

0 comments on commit 7e0d9de

Please sign in to comment.