Fetch dependencies for -Zbuild-std before entering the sandbox

This allows running `doc -Zbuild-std` from within the sandbox.
Previously, it would error out because cargo tried to download the
standard library's dependencies:

```
[2021-11-27T19:57:24Z INFO  rustwide::cmd] running `Command { std: "docker" "create" "-v" "/home/joshua/src/rust/docs.rs/.workspace/builds/gba-0.5.2/target:/opt/rustwide/target:rw,Z" "-v" "/home/joshua/src/rust/docs.rs/.workspace/builds/gba-0.5.2/source:/opt/rustwide/workdir:ro,Z" "-v" "/home/joshua/src/rust/docs.rs/.workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/home/joshua/src/rust/docs.rs/.workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "DOCS_RS=1" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "3221225472" "--user" "1000:1000" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux-micro" "/opt/rustwide/cargo-home/bin/cargo" "+nightly" "rustdoc" "--lib" "-Zrustdoc-map" "-Z" "unstable-options" "--config" "build.rustdocflags=[\"--cfg\", \"docs_rs\", \"-Z\", \"unstable-options\", \"--emit=invocation-specific\", \"--resource-suffix\", \"-20211126-1.58.0-nightly-6d246f0c8\", \"--static-root-path\", \"/\", \"--cap-lints\", \"warn\", \"--disable-per-crate-search\"]" "-Zunstable-options" "--config=doc.extern-map.registries.crates-io=\"https://docs.rs/{pkg_name}/{version}/thumbv4t-none-eabi\"" "-Zbuild-std" "--target" "thumbv4t-none-eabi", kill_on_drop: false }`
[2021-11-27T19:57:24Z INFO  rustwide::cmd] [stdout] fe2773f8a17ab13ce7b66c7221f91883a7b92b3bdeef7b30bf2ed55aa6b3d511
[2021-11-27T19:57:24Z INFO  rustwide::cmd] running `Command { std: "docker" "start" "-a" "fe2773f8a17ab13ce7b66c7221f91883a7b92b3bdeef7b30bf2ed55aa6b3d511", kill_on_drop: false }`
[2021-11-27T19:57:24Z INFO  rustwide::cmd] [stderr]  Downloading crates ...
[2021-11-27T19:57:24Z INFO  rustwide::cmd] [stderr] warning: spurious network error (2 tries remaining): [6] Couldn't resolve host name (Could not resolve host: crates.io)
[2021-11-27T19:57:24Z INFO  rustwide::cmd] [stderr] error: failed to download from `https://crates.io/api/v1/crates/libc/0.2.106/download`
[2021-11-27T19:57:24Z INFO  rustwide::cmd] [stderr]
[2021-11-27T19:57:24Z INFO  rustwide::cmd] [stderr] Caused by:
[2021-11-27T19:57:24Z INFO  rustwide::cmd] [stderr]   [6] Couldn't resolve host name (Could not resolve host: crates.io)
```

CHANGELOG.md

@@ -5,6 +5,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Builds in the sandbox can now use `-Zbuild-std` when networking is disabled.
+  Previously, this would try to fetch the standard library sources, which would
+  error when networking was blocked. Note that you will still need to run
+  `toolchain.add_component("rust-src")` before trying to use build-std.
+
 ## [0.15.2] - 2022-11-08
 
 ### Changed

src/prepare.rs

@@ -139,9 +139,18 @@ impl<'a> Prepare<'a> {
 
     fn fetch_deps(&mut self) -> Result<(), Error> {
         let mut missing_deps = false;
-        let res = Command::new(self.workspace, self.toolchain.cargo())
+        let mut cmd = Command::new(self.workspace, self.toolchain.cargo())
             .args(&["fetch", "--manifest-path", "Cargo.toml"])
-            .cd(self.source_dir)
+            .cd(self.source_dir);
+        // Pass `-Zbuild-std` in case a build in the sandbox wants to use it;
+        // build-std has to have the source for libstd's dependencies available.
+        if let Some(target) = self.workspace.fetch_build_std_dependencies() {
+            self.toolchain.add_component(self.workspace, "rust-src")?;
+            cmd = cmd
+                .args(&["-Zbuild-std", "--target", target])
+                .env("__CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS", "nightly");
+        }
+        let res = cmd
             .process_lines(&mut |line, _| {
                 if line.contains("failed to load source for dependency") {
                     missing_deps = true;

src/workspace.rs

@@ -27,6 +27,8 @@ pub struct WorkspaceBuilder {
     command_timeout: Option<Duration>,
     command_no_output_timeout: Option<Duration>,
     fetch_registry_index_during_builds: bool,
+    /// Stores the `--target` to fetch dependencies for.
+    fetch_build_std_dependencies: Option<String>,
     running_inside_docker: bool,
     fast_init: bool,
     rustup_profile: String,
@@ -45,6 +47,7 @@ impl WorkspaceBuilder {
             command_timeout: DEFAULT_COMMAND_TIMEOUT,
             command_no_output_timeout: DEFAULT_COMMAND_NO_OUTPUT_TIMEOUT,
             fetch_registry_index_during_builds: true,
+            fetch_build_std_dependencies: None,
             running_inside_docker: false,
             fast_init: false,
             rustup_profile: DEFAULT_RUSTUP_PROFILE.into(),
@@ -107,6 +110,18 @@ impl WorkspaceBuilder {
         self
     }
 
+    /// Enable or disable pre-fetching the dependencies for `-Z build-std` when preparing the sandbox (disabled by default).
+    ///
+    /// When this option is enabled, it is possible to use `-Zbuild-std` inside
+    /// the sandbox to build the standard library from source even when
+    /// networking is disabled.
+    #[cfg(any(feature = "unstable", doc))]
+    #[cfg_attr(docs_rs, doc(cfg(feature = "unstable")))]
+    pub fn fetch_build_std_dependencies(mut self, target: String) -> Self {
+        self.fetch_build_std_dependencies = Some(target);
+        self
+    }
+
     /// Enable or disable support for running Rustwide itself inside Docker (disabled by default).
     ///
     /// When support is enabled Rustwide will try to detect whether it's actually running inside a
@@ -160,6 +175,7 @@ impl WorkspaceBuilder {
                     command_timeout: self.command_timeout,
                     command_no_output_timeout: self.command_no_output_timeout,
                     fetch_registry_index_during_builds: self.fetch_registry_index_during_builds,
+                    fetch_build_std_dependencies: self.fetch_build_std_dependencies,
                     current_container: None,
                     rustup_profile: self.rustup_profile,
                 }),
@@ -183,6 +199,7 @@ struct WorkspaceInner {
     command_timeout: Option<Duration>,
     command_no_output_timeout: Option<Duration>,
     fetch_registry_index_during_builds: bool,
+    fetch_build_std_dependencies: Option<String>,
     current_container: Option<CurrentContainer>,
     rustup_profile: String,
 }
@@ -299,6 +316,10 @@ impl Workspace {
         self.inner.fetch_registry_index_during_builds
     }
 
+    pub(crate) fn fetch_build_std_dependencies(&self) -> Option<&str> {
+        self.inner.fetch_build_std_dependencies.as_deref()
+    }
+
     pub(crate) fn current_container(&self) -> Option<&CurrentContainer> {
         self.inner.current_container.as_ref()
     }

tests/buildtest/mod.rs

@@ -2,6 +2,8 @@ use failure::Error;
 use log::LevelFilter;
 use rustwide::cmd::{ProcessLinesActions, SandboxBuilder};
 
+use self::runner::Runner;
+
 #[macro_use]
 mod runner;
 mod inside_docker;
@@ -26,6 +28,36 @@ fn test_hello_world() {
     });
 }
 
+#[test]
+#[cfg(feature = "unstable")]
+fn test_fetch_build_std() {
+    use std::path::Path;
+
+    let target_file = Path::new(env!("OUT_DIR")).join("target");
+    let target = std::fs::read_to_string(target_file).unwrap();
+    let workspace = crate::utils::prepare_named_workspace("integration")
+        .unwrap()
+        .fetch_build_std_dependencies(target.clone())
+        .init()
+        .unwrap();
+    let run = Runner::new("hello-world", workspace).unwrap();
+    run.run(SandboxBuilder::new().enable_networking(false), |build| {
+        let storage = rustwide::logging::LogStorage::new(LevelFilter::Info);
+        rustwide::logging::capture(&storage, || -> Result<_, Error> {
+            build.cargo().env("RUSTC_BOOTSTRAP", "1")
+                 .args(&["run", "-Zbuild-std", "--target", &target])
+                 .run()?;
+            Ok(())
+        })?;
+
+        assert!(storage.to_string().contains("[stdout] Hello, world!\n"));
+        assert!(storage
+            .to_string()
+            .contains("[stdout] Hello, world again!\n"));
+        Ok(())
+    }).unwrap();
+}
+
 #[test]
 fn path_based_patch() {
     runner::run("path-based-patch", |run| {

tests/buildtest/runner.rs

@@ -3,7 +3,8 @@ use rustwide::{cmd::SandboxBuilder, Build, BuildBuilder, Crate, Toolchain, Works
 use std::path::Path;
 
 pub(crate) fn run(crate_name: &str, f: impl FnOnce(&mut Runner) -> Result<(), Error>) {
-    let mut runner = Runner::new(crate_name).unwrap();
+    let workspace = crate::utils::init_workspace().unwrap();
+    let mut runner = Runner::new(crate_name, workspace).unwrap();
     f(&mut runner).unwrap();
 }
 
@@ -15,8 +16,7 @@ pub(crate) struct Runner {
 }
 
 impl Runner {
-    fn new(crate_name: &str) -> Result<Self, Error> {
-        let workspace = crate::utils::init_workspace()?;
+    pub(crate) fn new(crate_name: &str, workspace: Workspace) -> Result<Self, Error> {
         let krate = Crate::local(
             &Path::new("tests")
                 .join("buildtest")

tests/utils/mod.rs

@@ -13,7 +13,7 @@ pub(crate) fn init_workspace() -> Result<Workspace, Error> {
     init_named_workspace("integration")
 }
 
-pub(crate) fn init_named_workspace(name: &str) -> Result<Workspace, Error> {
+pub(crate) fn prepare_named_workspace(name: &str) -> Result<WorkspaceBuilder, Error> {
     init_logs();
     let workspace_path = workspace_path(name);
     let mut builder = WorkspaceBuilder::new(&workspace_path, USER_AGENT).fast_init(true);
@@ -29,7 +29,11 @@ pub(crate) fn init_named_workspace(name: &str) -> Result<Workspace, Error> {
         )?);
     }
 
-    builder.init()
+    Ok(builder)
+}
+
+pub(crate) fn init_named_workspace(name: &str) -> Result<Workspace, Error> {
+    prepare_named_workspace(name)?.init()
 }
 
 fn init_logs() {