[WIP] Combine run-pass tests into a single crate

config.toml.example

@@ -290,6 +290,9 @@
 # desired in distributions, for example.
 #rpath = true
 
+# Merge Rust test files into larger units for faster testing
+#combine-tests = false
+
 # Suppresses extraneous output from tests to ensure the output of the test
 # harness is relatively clean.
 #quiet-tests = false

src/bootstrap/builder.rs

@@ -14,6 +14,7 @@ use std::collections::BTreeSet;
 use std::env;
 use std::fmt::Debug;
 use std::fs;
+use std::cmp;
 use std::hash::Hash;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
@@ -551,6 +552,7 @@ impl<'a> Builder<'a> {
         } else {
             stage = compiler.stage;
         }
+        let source_stage = cmp::max(stage, 1);
 
         let mut extra_args = env::var(&format!("RUSTFLAGS_STAGE_{}", stage)).unwrap_or_default();
         if stage != 0 {
@@ -598,7 +600,8 @@ impl<'a> Builder<'a> {
             cargo.env("RUSTC_ERROR_FORMAT", error_format);
         }
         if cmd != "build" && cmd != "check" {
-            cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(self.compiler(2, self.build.build)));
+            cargo.env("RUSTDOC_LIBDIR",
+                      self.rustc_libdir(self.compiler(source_stage, self.build.build)));
         }
 
         if mode != Mode::Tool {

src/bootstrap/config.rs

@@ -119,6 +119,7 @@ pub struct Config {
     pub low_priority: bool,
     pub channel: String,
     pub quiet_tests: bool,
+    pub combine_tests: bool,
     pub test_miri: bool,
     pub save_toolstates: Option<PathBuf>,
     pub print_step_timings: bool,
@@ -290,6 +291,7 @@ struct Rust {
     debug: Option<bool>,
     dist_src: Option<bool>,
     quiet_tests: Option<bool>,
+    combine_tests: Option<bool>,
     test_miri: Option<bool>,
     save_toolstates: Option<String>,
     codegen_backends: Option<Vec<String>>,
@@ -481,6 +483,7 @@ impl Config {
             set(&mut config.backtrace, rust.backtrace);
             set(&mut config.channel, rust.channel.clone());
             set(&mut config.rust_dist_src, rust.dist_src);
+            set(&mut config.combine_tests, rust.combine_tests);
             set(&mut config.quiet_tests, rust.quiet_tests);
             set(&mut config.test_miri, rust.test_miri);
             set(&mut config.wasm_syscall, rust.wasm_syscall);

src/bootstrap/configure.py

@@ -48,6 +48,7 @@ def v(*args):
 o("test-miri", "rust.test-miri", "run miri's test suite")
 o("debuginfo-tests", "rust.debuginfo-tests", "build tests with debugger metadata")
 o("quiet-tests", "rust.quiet-tests", "enable quieter output when running tests")
+o("combined-tests", "rust.combine-tests", "merge tests together when possible")
 o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds")
 o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds")
 o("local-rust", None, "use an installed rustc rather than downloading a snapshot")

src/bootstrap/lib.rs

@@ -113,7 +113,6 @@
 //! More documentation can be found in each respective module below, and you can
 //! also check out the `src/bootstrap/README.md` file for more information.
 
-#![deny(warnings)]
 #![feature(core_intrinsics)]
 #![feature(slice_concat_ext)]
 

src/bootstrap/test.rs

@@ -851,6 +851,10 @@ impl Step for Compiletest {
         cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target));
         cmd.arg("--rustc-path").arg(builder.rustc(compiler));
 
+        if builder.config.combine_tests && mode == "run-pass" {
+            cmd.arg("--combine");
+        }
+
         // Avoid depending on rustdoc when we don't need it.
         if mode == "rustdoc" || (mode == "run-make" && suite.ends_with("fulldeps")) {
             cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler.host));
@@ -1420,6 +1424,11 @@ impl Step for Crate {
             cargo.arg("--doc");
         }
 
+        if build.config.combine_tests {
+            cargo.env("RUSTDOC_COMBINE_TESTS", "1");
+            //cargo.arg("--doc"); -- RUNS ONLY DOCTESTS
+        }
+
         cargo.arg("-p").arg(krate);
 
         // The tests are going to run with the *target* libraries, so we need to

src/ci/docker/x86_64-gnu-llvm-3.9/Dockerfile

@@ -23,5 +23,6 @@ RUN sh /scripts/sccache.sh
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --llvm-root=/usr/lib/llvm-3.9 \
+      --enable-combined-tests \
       --enable-llvm-link-shared
 ENV RUST_CHECK_TARGET check

src/librustc/session/config.rs

@@ -1114,6 +1114,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "generate comments into the assembly (may change behavior)"),
     no_verify: bool = (false, parse_bool, [TRACKED],
         "skip LLVM verification"),
+    combine_tests: bool = (false, parse_bool, [TRACKED],
+        "pretend immediate submodules of the crate are crates themselves"),
     borrowck_stats: bool = (false, parse_bool, [UNTRACKED],
         "gather borrowck statistics"),
     no_landing_pads: bool = (false, parse_bool, [TRACKED],

src/librustc/session/mod.rs

@@ -1095,7 +1095,8 @@ pub fn build_session_(
     };
     let target_cfg = config::build_target_config(&sopts, &span_diagnostic);
 
-    let p_s = parse::ParseSess::with_span_handler(span_diagnostic, codemap);
+    let mut p_s = parse::ParseSess::with_span_handler(span_diagnostic, codemap);
+    p_s.combine_tests = sopts.debugging_opts.combine_tests;
     let default_sysroot = match sopts.maybe_sysroot {
         Some(_) => None,
         None => Some(filesearch::get_or_default_sysroot()),

src/librustc_driver/driver.rs

@@ -847,12 +847,30 @@ pub fn phase_2_configure_and_expand_inner<'a, F>(sess: &'a Session,
         });
     }
 
+    if sess.opts.debugging_opts.ast_json_noexpand {
+        println!("\n\n\n\n\n\n\n\nPRE EXPAND:");
+        println!("{}", json::as_json(&krate));
+    }
+
     krate = time(sess, "creating allocators", || {
         allocator::expand::modify(&sess.parse_sess,
                                   &mut resolver,
                                   krate,
                                   sess.diagnostic())
     });
+/*
+    if sess.opts.debugging_opts.submodules_crate_like {
+        krate = time(sess,
+            "Pretending submodules are crates",
+            || rustc_passes::submodules_crate_like::modify_crate(krate));
+        eprintln!("expanded module");
+    }
+*/
+    if sess.opts.debugging_opts.ast_json_noexpand {
+        println!("\n\n\n\n\n\n\n\nPOST EXPAND: {:#?}", krate);
+        println!("\n\n\n\n\n\n\n\nPOST EXPAND:");
+        println!("{}", json::as_json(&krate));
+    }
 
     after_expand(&krate)?;
 
@@ -884,7 +902,8 @@ pub fn phase_2_configure_and_expand_inner<'a, F>(sess: &'a Session,
                                               &sess.parse_sess,
                                               &sess.features_untracked(),
                                               &attributes,
-                                              sess.opts.unstable_features);
+                                              sess.opts.unstable_features,
+                                              sess.opts.debugging_opts.edition);
         })
     })?;
 

src/librustc_driver/lib.rs

@@ -901,8 +901,8 @@ impl<'a> CompilerCalls<'a> for RustcDefaultCalls {
         }
 
         if sess.opts.debugging_opts.parse_only ||
-           sess.opts.debugging_opts.show_span.is_some() ||
-           sess.opts.debugging_opts.ast_json_noexpand {
+           sess.opts.debugging_opts.show_span.is_some()/* ||
+           sess.opts.debugging_opts.ast_json_noexpand*/ {
             control.after_parse.stop = Compilation::Stop;
         }
 

src/librustdoc/Cargo.toml

@@ -10,3 +10,4 @@ path = "lib.rs"
 [dependencies]
 pulldown-cmark = { version = "0.1.2", default-features = false }
 tempdir = "0.3"
+itertools = "0.7.6"