From 684aa2c9d11d9e818ea65f7e4ebd194b97318ac7 Mon Sep 17 00:00:00 2001
From: David Koloski <djkoloski@gmail.com>
Date: Wed, 6 Dec 2023 18:25:13 +0000
Subject: [PATCH] Add support for shell argfiles
---
Cargo.lock | 1 +
compiler/rustc_driver_impl/Cargo.toml | 1 +
compiler/rustc_driver_impl/src/args.rs | 107 +++++++++++++++---
compiler/rustc_interface/src/tests.rs | 1 +
compiler/rustc_session/src/options.rs | 2 +
.../src/compiler-flags/shell-argfiles.md | 11 ++
src/tools/tidy/src/deps.rs | 1 +
src/tools/tidy/src/ui_tests.rs | 6 +-
.../shell-argfiles-badquotes-windows.rs | 11 ++
.../shell-argfiles-badquotes-windows.stderr | 2 +
.../shell-argfiles-badquotes.args | 1 +
.../shell-argfiles-badquotes.rs | 12 ++
.../shell-argfiles-badquotes.stderr | 2 +
.../shell-argfiles-via-argfile-shell.args | 1 +
.../shell-argfiles-via-argfile.args | 1 +
.../shell-argfiles-via-argfile.rs | 10 ++
tests/ui/shell-argfiles/shell-argfiles.args | 3 +
tests/ui/shell-argfiles/shell-argfiles.rs | 19 ++++
18 files changed, 175 insertions(+), 17 deletions(-)
create mode 100644 src/doc/unstable-book/src/compiler-flags/shell-argfiles.md
create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs
create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr
create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes.args
create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes.rs
create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr
create mode 100644 tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args
create mode 100644 tests/ui/shell-argfiles/shell-argfiles-via-argfile.args
create mode 100644 tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs
create mode 100644 tests/ui/shell-argfiles/shell-argfiles.args
create mode 100644 tests/ui/shell-argfiles/shell-argfiles.rs
diff --git a/Cargo.lock b/Cargo.lock
index b8192e333fe91..c7fa0b5675286 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3738,6 +3738,7 @@ dependencies = [
"rustc_trait_selection",
"rustc_ty_utils",
"serde_json",
+ "shlex",
"time",
"tracing",
"windows",
diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml
index 490429845538d..97a7dfef3b395 100644
--- a/compiler/rustc_driver_impl/Cargo.toml
+++ b/compiler/rustc_driver_impl/Cargo.toml
@@ -50,6 +50,7 @@ rustc_target = { path = "../rustc_target" }
rustc_trait_selection = { path = "../rustc_trait_selection" }
rustc_ty_utils = { path = "../rustc_ty_utils" }
serde_json = "1.0.59"
+shlex = "1.0"
time = { version = "0.3", default-features = false, features = ["alloc", "formatting"] }
tracing = { version = "0.1.35" }
# tidy-alphabetical-end
diff --git a/compiler/rustc_driver_impl/src/args.rs b/compiler/rustc_driver_impl/src/args.rs
index b7407f5a508e4..5dfd37a6da4de 100644
--- a/compiler/rustc_driver_impl/src/args.rs
+++ b/compiler/rustc_driver_impl/src/args.rs
@@ -5,18 +5,92 @@ use std::io;
use rustc_session::EarlyDiagCtxt;
-fn arg_expand(arg: String) -> Result<Vec<String>, Error> {
- if let Some(path) = arg.strip_prefix('@') {
- let file = match fs::read_to_string(path) {
- Ok(file) => file,
- Err(ref err) if err.kind() == io::ErrorKind::InvalidData => {
- return Err(Error::Utf8Error(Some(path.to_string())));
+/// Expands argfiles in command line arguments.
+#[derive(Default)]
+struct Expander {
+ shell_argfiles: bool,
+ next_is_unstable_option: bool,
+ expanded: Vec<String>,
+}
+
+impl Expander {
+ /// Handles the next argument. If the argument is an argfile, it is expanded
+ /// inline.
+ fn arg(&mut self, arg: &str) -> Result<(), Error> {
+ if let Some(argfile) = arg.strip_prefix('@') {
+ match argfile.split_once(':') {
+ Some(("shell", path)) if self.shell_argfiles => {
+ shlex::split(&Self::read_file(path)?)
+ .ok_or_else(|| Error::ShellParseError(path.to_string()))?
+ .into_iter()
+ .for_each(|arg| self.push(arg));
+ }
+ _ => {
+ let contents = Self::read_file(argfile)?;
+ contents.lines().for_each(|arg| self.push(arg.to_string()));
+ }
+ }
+ } else {
+ self.push(arg.to_string());
+ }
+
+ Ok(())
+ }
+
+ /// Adds a command line argument verbatim with no argfile expansion.
+ fn push(&mut self, arg: String) {
+ // Unfortunately, we have to do some eager argparsing to handle unstable
+ // options which change the behavior of argfile arguments.
+ //
+ // Normally, all of the argfile arguments (e.g. `@args.txt`) are
+ // expanded into our arguments list *and then* the whole list of
+ // arguments are passed on to be parsed. However, argfile parsing
+ // options like `-Zshell_argfiles` need to change the behavior of that
+ // argument expansion. So we have to do a little parsing on our own here
+ // instead of leaning on the existing logic.
+ //
+ // All we care about are unstable options, so we parse those out and
+ // look for any that affect how we expand argfiles. This argument
+ // inspection is very conservative; we only change behavior when we see
+ // exactly the options we're looking for and everything gets passed
+ // through.
+
+ if self.next_is_unstable_option {
+ self.inspect_unstable_option(&arg);
+ self.next_is_unstable_option = false;
+ } else if let Some(unstable_option) = arg.strip_prefix("-Z") {
+ if unstable_option.is_empty() {
+ self.next_is_unstable_option = true;
+ } else {
+ self.inspect_unstable_option(unstable_option);
+ }
+ }
+
+ self.expanded.push(arg);
+ }
+
+ /// Consumes the `Expander`, returning the expanded arguments.
+ fn finish(self) -> Vec<String> {
+ self.expanded
+ }
+
+ /// Parses any relevant unstable flags specified on the command line.
+ fn inspect_unstable_option(&mut self, option: &str) {
+ match option {
+ "shell-argfiles" => self.shell_argfiles = true,
+ _ => (),
+ }
+ }
+
+ /// Reads the contents of a file as UTF-8.
+ fn read_file(path: &str) -> Result<String, Error> {
+ fs::read_to_string(path).map_err(|e| {
+ if e.kind() == io::ErrorKind::InvalidData {
+ Error::Utf8Error(Some(path.to_string()))
+ } else {
+ Error::IOError(path.to_string(), e)
}
- Err(err) => return Err(Error::IOError(path.to_string(), err)),
- };
- Ok(file.lines().map(ToString::to_string).collect())
- } else {
- Ok(vec![arg])
+ })
}
}
@@ -24,20 +98,20 @@ fn arg_expand(arg: String) -> Result<Vec<String>, Error> {
/// If this function is intended to be used with command line arguments,
/// `argv[0]` must be removed prior to calling it manually.
pub fn arg_expand_all(early_dcx: &EarlyDiagCtxt, at_args: &[String]) -> Vec<String> {
- let mut args = Vec::new();
+ let mut expander = Expander::default();
for arg in at_args {
- match arg_expand(arg.clone()) {
- Ok(arg) => args.extend(arg),
- Err(err) => early_dcx.early_fatal(format!("Failed to load argument file: {err}")),
+ if let Err(err) = expander.arg(arg) {
+ early_dcx.early_fatal(format!("Failed to load argument file: {err}"));
}
}
- args
+ expander.finish()
}
#[derive(Debug)]
pub enum Error {
Utf8Error(Option<String>),
IOError(String, io::Error),
+ ShellParseError(String),
}
impl fmt::Display for Error {
@@ -46,6 +120,7 @@ impl fmt::Display for Error {
Error::Utf8Error(None) => write!(fmt, "Utf8 error"),
Error::Utf8Error(Some(path)) => write!(fmt, "Utf8 error in {path}"),
Error::IOError(path, err) => write!(fmt, "IO Error: {path}: {err}"),
+ Error::ShellParseError(path) => write!(fmt, "Invalid shell-style arguments in {path}"),
}
}
}
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 75410db1e364c..7b5de4cc11720 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -700,6 +700,7 @@ fn test_unstable_options_tracking_hash() {
untracked!(query_dep_graph, true);
untracked!(self_profile, SwitchWithOptPath::Enabled(None));
untracked!(self_profile_events, Some(vec![String::new()]));
+ untracked!(shell_argfiles, true);
untracked!(span_debug, true);
untracked!(span_free_formats, true);
untracked!(temps_dir, Some(String::from("abc")));
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 0b0b67ef890b0..21113c298f04a 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1875,6 +1875,8 @@ written to standard error output)"),
query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm, artifact-sizes"),
share_generics: Option<bool> = (None, parse_opt_bool, [TRACKED],
"make the current crate share its generic instantiations"),
+ shell_argfiles: bool = (false, parse_bool, [UNTRACKED],
+ "allow argument files to be specified with POSIX \"shell-style\" argument quoting"),
show_span: Option<String> = (None, parse_opt_string, [TRACKED],
"show spans for compiler debugging (expr|pat|ty)"),
simulate_remapped_rust_src_base: Option<PathBuf> = (None, parse_opt_pathbuf, [TRACKED],
diff --git a/src/doc/unstable-book/src/compiler-flags/shell-argfiles.md b/src/doc/unstable-book/src/compiler-flags/shell-argfiles.md
new file mode 100644
index 0000000000000..4f3c780972de5
--- /dev/null
+++ b/src/doc/unstable-book/src/compiler-flags/shell-argfiles.md
@@ -0,0 +1,11 @@
+# `shell-argfiles`
+
+--------------------
+
+The `-Zshell-argfiles` compiler flag allows argfiles to be parsed using POSIX
+"shell-style" quoting. When enabled, the compiler will use `shlex` to parse the
+arguments from argfiles specified with `@shell:<path>`.
+
+Because this feature controls the parsing of input arguments, the
+`-Zshell-argfiles` flag must be present before the argument specifying the
+shell-style arguemnt file.
diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index 3c00027b9fdc9..62d48315d434e 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -325,6 +325,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
"sha1",
"sha2",
"sharded-slab",
+ "shlex",
"smallvec",
"snap",
"stable_deref_trait",
diff --git a/src/tools/tidy/src/ui_tests.rs b/src/tools/tidy/src/ui_tests.rs
index b4745d4883c55..ab0e647e13043 100644
--- a/src/tools/tidy/src/ui_tests.rs
+++ b/src/tools/tidy/src/ui_tests.rs
@@ -11,7 +11,7 @@ use std::path::{Path, PathBuf};
const ENTRY_LIMIT: usize = 900;
// FIXME: The following limits should be reduced eventually.
const ISSUES_ENTRY_LIMIT: usize = 1849;
-const ROOT_ENTRY_LIMIT: usize = 867;
+const ROOT_ENTRY_LIMIT: usize = 868;
const EXPECTED_TEST_FILE_EXTENSIONS: &[&str] = &[
"rs", // test source files
@@ -36,6 +36,10 @@ const EXTENSION_EXCEPTION_PATHS: &[&str] = &[
"tests/ui/unused-crate-deps/test.mk", // why would you use make
"tests/ui/proc-macro/auxiliary/included-file.txt", // more include
"tests/ui/invalid/foo.natvis.xml", // sample debugger visualizer
+ "tests/ui/shell-argfiles/shell-argfiles.args", // passing args via a file
+ "tests/ui/shell-argfiles/shell-argfiles-badquotes.args", // passing args via a file
+ "tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args", // passing args via a file
+ "tests/ui/shell-argfiles/shell-argfiles-via-argfile.args", // passing args via a file
];
fn check_entries(tests_path: &Path, bad: &mut bool) {
diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs b/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs
new file mode 100644
index 0000000000000..800735cf3a768
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs
@@ -0,0 +1,11 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// Path replacement in .stderr files (i.e. `$DIR`) doesn't handle mixed path
+// separators. This test uses backslash as the path separator for the command
+// line arguments and is only run on windows.
+//
+// only-windows
+// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}\shell-argfiles\shell-argfiles-badquotes.args
+
+fn main() {
+}
diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr b/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr
new file mode 100644
index 0000000000000..14adb1f740abb
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr
@@ -0,0 +1,2 @@
+error: Failed to load argument file: Invalid shell-style arguments in $DIR/shell-argfiles-badquotes.args
+
diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes.args b/tests/ui/shell-argfiles/shell-argfiles-badquotes.args
new file mode 100644
index 0000000000000..c0d531adf3ffb
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes.args
@@ -0,0 +1 @@
+"--cfg" "unquoted_set
diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes.rs b/tests/ui/shell-argfiles/shell-argfiles-badquotes.rs
new file mode 100644
index 0000000000000..f9160143a0410
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes.rs
@@ -0,0 +1,12 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// Path replacement in .stderr files (i.e. `$DIR`) doesn't handle mixed path
+// separators. We have a duplicated version of this test that uses backslash as
+// the path separator for the command line arguments that is only run on
+// windows.
+//
+// ignore-windows
+// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}/shell-argfiles/shell-argfiles-badquotes.args
+
+fn main() {
+}
diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr b/tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr
new file mode 100644
index 0000000000000..14adb1f740abb
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr
@@ -0,0 +1,2 @@
+error: Failed to load argument file: Invalid shell-style arguments in $DIR/shell-argfiles-badquotes.args
+
diff --git a/tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args b/tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args
new file mode 100644
index 0000000000000..4e66d5a039529
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args
@@ -0,0 +1 @@
+"--cfg" "shell_args_set"
\ No newline at end of file
diff --git a/tests/ui/shell-argfiles/shell-argfiles-via-argfile.args b/tests/ui/shell-argfiles/shell-argfiles-via-argfile.args
new file mode 100644
index 0000000000000..d0af54e24e33c
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles-via-argfile.args
@@ -0,0 +1 @@
+-Zshell-argfiles
\ No newline at end of file
diff --git a/tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs b/tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs
new file mode 100644
index 0000000000000..d71e3218f53b8
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs
@@ -0,0 +1,10 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-pass
+// compile-flags: @{{src-base}}/shell-argfiles/shell-argfiles-via-argfile.args @shell:{{src-base}}/shell-argfiles/shell-argfiles-via-argfile-shell.args
+
+#[cfg(not(shell_args_set))]
+compile_error!("shell_args_set not set");
+
+fn main() {
+}
diff --git a/tests/ui/shell-argfiles/shell-argfiles.args b/tests/ui/shell-argfiles/shell-argfiles.args
new file mode 100644
index 0000000000000..e5bb4b807ec4d
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles.args
@@ -0,0 +1,3 @@
+--cfg unquoted_set
+'--cfg' 'single_quoted_set'
+"--cfg" "double_quoted_set"
diff --git a/tests/ui/shell-argfiles/shell-argfiles.rs b/tests/ui/shell-argfiles/shell-argfiles.rs
new file mode 100644
index 0000000000000..9bc252ea628a9
--- /dev/null
+++ b/tests/ui/shell-argfiles/shell-argfiles.rs
@@ -0,0 +1,19 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-pass
+// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}/shell-argfiles/shell-argfiles.args
+
+#[cfg(not(cmdline_set))]
+compile_error!("cmdline_set not set");
+
+#[cfg(not(unquoted_set))]
+compile_error!("unquoted_set not set");
+
+#[cfg(not(single_quoted_set))]
+compile_error!("single_quoted_set not set");
+
+#[cfg(not(double_quoted_set))]
+compile_error!("double_quoted_set not set");
+
+fn main() {
+}