Use response files for rustc

src/cargo/util/process_builder.rs

@@ -2,12 +2,14 @@ use std::collections::HashMap;
 use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fmt;
+use std::io::Write;
 use std::path::Path;
-use std::process::{Command, Output, Stdio};
+use std::process::{Child, Command, ExitStatus, Output, Stdio};
 
 use failure::Fail;
 use jobserver::Client;
 use shell_escape::escape;
+use tempfile::{NamedTempFile, TempPath};
 
 use crate::util::{process_error, read2, CargoResult, CargoResultExt};
 
@@ -151,8 +153,7 @@ impl ProcessBuilder {
 
     /// Runs the process, waiting for completion, and mapping non-success exit codes to an error.
     pub fn exec(&self) -> CargoResult<()> {
-        let mut command = self.build_command();
-        let exit = command.status().chain_err(|| {
+        let exit = self.status().chain_err(|| {
             process_error(&format!("could not execute process {}", self), None, None)
         })?;
 
@@ -189,9 +190,7 @@ impl ProcessBuilder {
 
     /// Executes the process, returning the stdio output, or an error if non-zero exit status.
     pub fn exec_with_output(&self) -> CargoResult<Output> {
-        let mut command = self.build_command();
-
-        let output = command.output().chain_err(|| {
+        let output = self.output().chain_err(|| {
             process_error(&format!("could not execute process {}", self), None, None)
         })?;
 
@@ -225,14 +224,13 @@ impl ProcessBuilder {
         let mut stdout = Vec::new();
         let mut stderr = Vec::new();
 
-        let mut cmd = self.build_command();
-        cmd.stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .stdin(Stdio::null());
-
         let mut callback_error = None;
-        let status = (|| {
-            let mut child = cmd.spawn()?;
+        let status = (|| -> CargoResult<ExitStatus> {
+            let (mut child, _response_file) = self.spawn_with(|cmd| {
+                cmd.stdout(Stdio::piped())
+                    .stderr(Stdio::piped())
+                    .stdin(Stdio::null());
+            })?;
             let out = child.stdout.take().unwrap();
             let err = child.stderr.take().unwrap();
             read2(out, err, &mut |is_out, data, eof| {
@@ -273,7 +271,7 @@ impl ProcessBuilder {
                     data.drain(..idx);
                 }
             })?;
-            child.wait()
+            Ok(child.wait()?)
         })()
         .chain_err(|| process_error(&format!("could not execute process {}", self), None, None))?;
         let output = Output {
@@ -329,6 +327,81 @@ impl ProcessBuilder {
         }
         command
     }
+
+    /// Converts `ProcessBuilder` into a `std::process::Command` and a rustc style response
+    /// file.  Also handles the jobserver, if present.
+    pub fn build_command_and_response_file(&self) -> CargoResult<(Command, TempPath)> {
+        // The rust linker also jumps through similar hoops, although with a different
+        // of response file, which this borrows from.  Some references:
+        // https://github.com/rust-lang/rust/blob/ef92009c1dbe2750f1d24a6619b827721fb49749/src/librustc_codegen_ssa/back/link.rs#L935
+        // https://github.com/rust-lang/rust/blob/ef92009c1dbe2750f1d24a6619b827721fb49749/src/librustc_codegen_ssa/back/command.rs#L135
+
+        let mut command = Command::new(&self.program);
+        if let Some(cwd) = self.get_cwd() {
+            command.current_dir(cwd);
+        }
+        // cmd.exe can handle up to 8k work of args, this leaves some headroom if using a .cmd rustc wrapper.
+        let mut cmd_remaining: usize = 1024 * 6;
+        let mut response_file = NamedTempFile::new()?;
+        for arg in &self.args {
+            cmd_remaining = cmd_remaining.saturating_sub(arg.len());
+            if cmd_remaining > 0 {
+                command.arg(arg);
+            } else if let Some(arg) = arg.to_str() {
+                writeln!(response_file, "{}", arg)?;
+            } else {
+                failure::bail!("argument {:?} contains invalid unicode", arg);
+            }
+        }
+        let mut arg = OsString::from("@");
+        arg.push(response_file.path());
+        command.arg(arg);
+        for (k, v) in &self.env {
+            match *v {
+                Some(ref v) => {
+                    command.env(k, v);
+                }
+                None => {
+                    command.env_remove(k);
+                }
+            }
+        }
+        if let Some(ref c) = self.jobserver {
+            c.configure(&mut command);
+        }
+        Ok((command, response_file.into_temp_path()))
+    }
+
+    fn status(&self) -> CargoResult<ExitStatus> {
+        let (mut child, _response_file) = self.spawn_with(|_cmd| {})?;
+        Ok(child.wait()?)
+    }
+
+    fn output(&self) -> CargoResult<Output> {
+        let (child, _response_file) = self.spawn_with(|cmd| {
+            cmd.stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .stdin(Stdio::null());
+        })?;
+        Ok(child.wait_with_output()?)
+    }
+
+    fn spawn_with(
+        &self,
+        mut modify_command: impl FnMut(&mut Command),
+    ) -> CargoResult<(Child, Option<TempPath>)> {
+        let mut command = self.build_command();
+        modify_command(&mut command);
+        match command.spawn() {
+            Ok(child) => Ok((child, None)),
+            Err(ref err) if imp::command_line_too_big(err) => {
+                let (mut command, response_file) = self.build_command_and_response_file()?;
+                modify_command(&mut command);
+                Ok((command.spawn()?, Some(response_file)))
+            }
+            Err(other) => Err(other.into()),
+        }
+    }
 }
 
 /// A helper function to create a `ProcessBuilder`.
@@ -360,13 +433,18 @@ mod imp {
             ))
             .into())
     }
+
+    pub fn command_line_too_big(err: &std::io::Error) -> bool {
+        err.raw_os_error() == Some(libc::E2BIG)
+    }
 }
 
 #[cfg(windows)]
 mod imp {
     use crate::util::{process_error, ProcessBuilder};
     use crate::CargoResult;
     use winapi::shared::minwindef::{BOOL, DWORD, FALSE, TRUE};
+    use winapi::shared::winerror::ERROR_FILENAME_EXCED_RANGE;
     use winapi::um::consoleapi::SetConsoleCtrlHandler;
 
     unsafe extern "system" fn ctrlc_handler(_: DWORD) -> BOOL {
@@ -384,4 +462,8 @@ mod imp {
         // Just execute the process as normal.
         process_builder.exec()
     }
+
+    pub fn command_line_too_big(err: &std::io::Error) -> bool {
+        err.raw_os_error() == Some(ERROR_FILENAME_EXCED_RANGE as i32)
+    }
 }