From 8c2ca6c5fd478e9e3f8887d38eaa2fd83cb75905 Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 16 Feb 2018 02:48:09 +0800 Subject: [PATCH] Revert "Auto merge of #48105 - Mark-Simulacrum:exclude-paths, r=alexcrichton" This reverts commit c83fa5d91c3b16459ab7b87c48ed18bd059a23af, reversing changes made to 90759befe0234b20ecf81edbbff353b85419d7e9. --- src/bootstrap/builder.rs | 122 ++------- src/bootstrap/check.rs | 6 +- src/bootstrap/compile.rs | 10 +- src/bootstrap/config.rs | 2 - src/bootstrap/dist.rs | 25 ++ src/bootstrap/doc.rs | 2 +- src/bootstrap/flags.rs | 20 +- src/bootstrap/lib.rs | 37 ++- src/bootstrap/native.rs | 4 +- src/bootstrap/test.rs | 543 ++++++++++++++++++--------------------- 10 files changed, 327 insertions(+), 444 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 001ae7246fdf2..03630dfbed3e0 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -95,7 +95,7 @@ pub struct RunConfig<'a> { pub builder: &'a Builder<'a>, pub host: Interned, pub target: Interned, - pub path: PathBuf, + pub path: Option<&'a Path>, } struct StepDescription { @@ -105,32 +105,6 @@ struct StepDescription { only_build: bool, should_run: fn(ShouldRun) -> ShouldRun, make_run: fn(RunConfig), - name: &'static str, -} - -#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] -struct PathSet { - set: BTreeSet, -} - -impl PathSet { - fn empty() -> PathSet { - PathSet { set: BTreeSet::new() } - } - - fn one>(path: P) -> PathSet { - let mut set = BTreeSet::new(); - set.insert(path.into()); - PathSet { set } - } - - fn has(&self, needle: &Path) -> bool { - self.set.iter().any(|p| p.ends_with(needle)) - } - - fn path(&self, builder: &Builder) -> PathBuf { - self.set.iter().next().unwrap_or(&builder.build.src).to_path_buf() - } } impl StepDescription { @@ -142,18 +116,10 @@ impl StepDescription { only_build: S::ONLY_BUILD, should_run: S::should_run, make_run: S::make_run, - name: unsafe { ::std::intrinsics::type_name::() }, } } - fn maybe_run(&self, builder: &Builder, pathset: &PathSet) { - if builder.config.exclude.iter().any(|e| pathset.has(e)) { - eprintln!("Skipping {:?} because it is excluded", pathset); - return; - } else if !builder.config.exclude.is_empty() { - eprintln!("{:?} not skipped for {:?} -- not in {:?}", pathset, - self.name, builder.config.exclude); - } + fn maybe_run(&self, builder: &Builder, path: Option<&Path>) { let build = builder.build; let hosts = if self.only_build_targets || self.only_build { build.build_triple() @@ -178,7 +144,7 @@ impl StepDescription { for target in targets { let run = RunConfig { builder, - path: pathset.path(builder), + path, host: *host, target: *target, }; @@ -191,28 +157,19 @@ impl StepDescription { let should_runs = v.iter().map(|desc| { (desc.should_run)(ShouldRun::new(builder)) }).collect::>(); - - // sanity checks on rules - for (desc, should_run) in v.iter().zip(&should_runs) { - assert!(!should_run.paths.is_empty(), - "{:?} should have at least one pathset", desc.name); - } - if paths.is_empty() { for (desc, should_run) in v.iter().zip(should_runs) { if desc.default && should_run.is_really_default { - for pathset in &should_run.paths { - desc.maybe_run(builder, pathset); - } + desc.maybe_run(builder, None); } } } else { for path in paths { let mut attempted_run = false; for (desc, should_run) in v.iter().zip(&should_runs) { - if let Some(pathset) = should_run.pathset_for_path(path) { + if should_run.run(path) { attempted_run = true; - desc.maybe_run(builder, pathset); + desc.maybe_run(builder, Some(path)); } } @@ -228,7 +185,7 @@ impl StepDescription { pub struct ShouldRun<'a> { pub builder: &'a Builder<'a>, // use a BTreeSet to maintain sort order - paths: BTreeSet, + paths: BTreeSet, // If this is a default rule, this is an additional constraint placed on // it's run. Generally something like compiler docs being enabled. @@ -249,46 +206,25 @@ impl<'a> ShouldRun<'a> { self } - // Unlike `krate` this will create just one pathset. As such, it probably shouldn't actually - // ever be used, but as we transition to having all rules properly handle passing krate(...) by - // actually doing something different for every crate passed. - pub fn all_krates(mut self, name: &str) -> Self { - let mut set = BTreeSet::new(); - for krate in self.builder.in_tree_crates(name) { - set.insert(PathBuf::from(&krate.path)); - } - self.paths.insert(PathSet { set }); - self - } - pub fn krate(mut self, name: &str) -> Self { - for krate in self.builder.in_tree_crates(name) { - self.paths.insert(PathSet::one(&krate.path)); + for (_, krate_path) in self.builder.crates(name) { + self.paths.insert(PathBuf::from(krate_path)); } self } - // single, non-aliased path - pub fn path(self, path: &str) -> Self { - self.paths(&[path]) - } - - // multiple aliases for the same job - pub fn paths(mut self, paths: &[&str]) -> Self { - self.paths.insert(PathSet { - set: paths.iter().map(PathBuf::from).collect(), - }); + pub fn path(mut self, path: &str) -> Self { + self.paths.insert(PathBuf::from(path)); self } // allows being more explicit about why should_run in Step returns the value passed to it - pub fn never(mut self) -> ShouldRun<'a> { - self.paths.insert(PathSet::empty()); + pub fn never(self) -> ShouldRun<'a> { self } - fn pathset_for_path(&self, path: &Path) -> Option<&PathSet> { - self.paths.iter().find(|pathset| pathset.has(path)) + fn run(&self, path: &Path) -> bool { + self.paths.iter().any(|p| path.ends_with(p)) } } @@ -318,23 +254,19 @@ impl<'a> Builder<'a> { tool::RustInstaller, tool::Cargo, tool::Rls, tool::Rustdoc, tool::Clippy, native::Llvm, tool::Rustfmt, tool::Miri), Kind::Check => describe!(check::Std, check::Test, check::Rustc), - Kind::Test => describe!(test::Tidy, test::Bootstrap, test::Ui, test::RunPass, - test::CompileFail, test::ParseFail, test::RunFail, test::RunPassValgrind, - test::MirOpt, test::Codegen, test::CodegenUnits, test::Incremental, test::Debuginfo, - test::UiFullDeps, test::RunPassFullDeps, test::RunFailFullDeps, - test::CompileFailFullDeps, test::IncrementalFullDeps, test::Rustdoc, test::Pretty, - test::RunPassPretty, test::RunFailPretty, test::RunPassValgrindPretty, - test::RunPassFullDepsPretty, test::RunFailFullDepsPretty, test::RunMake, - test::Crate, test::CrateLibrustc, test::Rustdoc, test::Linkcheck, test::Cargotest, - test::Cargo, test::Rls, test::Docs, test::ErrorIndex, test::Distcheck, - test::Rustfmt, test::Miri, test::Clippy, test::RustdocJS, test::RustdocTheme), + Kind::Test => describe!(test::Tidy, test::Bootstrap, test::DefaultCompiletest, + test::HostCompiletest, test::Crate, test::CrateLibrustc, test::Rustdoc, + test::Linkcheck, test::Cargotest, test::Cargo, test::Rls, test::Docs, + test::ErrorIndex, test::Distcheck, test::Rustfmt, test::Miri, test::Clippy, + test::RustdocJS, test::RustdocTheme), Kind::Bench => describe!(test::Crate, test::CrateLibrustc), Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook, doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon, doc::Reference, doc::Rustdoc, doc::RustByExample, doc::CargoBook), Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts, dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo, - dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign), + dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign, + dist::DontDistWithMiriEnabled), Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls, install::Rustfmt, install::Analysis, install::Src, install::Rustc), } @@ -365,10 +297,8 @@ impl<'a> Builder<'a> { should_run = (desc.should_run)(should_run); } let mut help = String::from("Available paths:\n"); - for pathset in should_run.paths { - for path in pathset.set { - help.push_str(format!(" ./x.py {} {}\n", subcommand, path.display()).as_str()); - } + for path in should_run.paths { + help.push_str(format!(" ./x.py {} {}\n", subcommand, path.display()).as_str()); } Some(help) } @@ -393,12 +323,6 @@ impl<'a> Builder<'a> { stack: RefCell::new(Vec::new()), }; - if kind == Kind::Dist { - assert!(!build.config.test_miri, "Do not distribute with miri enabled.\n\ - The distributed libraries would include all MIR (increasing binary size). - The distributed MIR would include validation statements."); - } - StepDescription::run(&Builder::get_step_descriptions(builder.kind), &builder, paths); } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 767ee4016c6f1..ede403491d7fc 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -26,7 +26,7 @@ impl Step for Std { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.all_krates("std") + run.path("src/libstd").krate("std") } fn make_run(run: RunConfig) { @@ -67,7 +67,7 @@ impl Step for Rustc { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.all_krates("rustc-main") + run.path("src/librustc").krate("rustc-main") } fn make_run(run: RunConfig) { @@ -114,7 +114,7 @@ impl Step for Test { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.all_krates("test") + run.path("src/libtest").krate("test") } fn make_run(run: RunConfig) { diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 2dcc0e0e7cd9f..1d5e11c5d6d41 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -48,7 +48,7 @@ impl Step for Std { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.all_krates("std") + run.path("src/libstd").krate("std") } fn make_run(run: RunConfig) { @@ -320,7 +320,7 @@ impl Step for Test { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.all_krates("test") + run.path("src/libtest").krate("test") } fn make_run(run: RunConfig) { @@ -436,7 +436,7 @@ impl Step for Rustc { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.all_krates("rustc-main") + run.path("src/librustc").krate("rustc-main") } fn make_run(run: RunConfig) { @@ -593,7 +593,7 @@ impl Step for CodegenBackend { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.all_krates("rustc_trans") + run.path("src/librustc_trans") } fn make_run(run: RunConfig) { @@ -828,7 +828,7 @@ impl Step for Assemble { type Output = Compiler; fn should_run(run: ShouldRun) -> ShouldRun { - run.all_krates("rustc-main") + run.path("src/rustc") } /// Prepare a new compiler from the artifacts in `stage` diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 812ca6d64fb6a..4f4fd14ae8cab 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -56,7 +56,6 @@ pub struct Config { pub sanitizers: bool, pub profiler: bool, pub ignore_git: bool, - pub exclude: Vec, pub run_host_only: bool, @@ -312,7 +311,6 @@ impl Config { let flags = Flags::parse(&args); let file = flags.config.clone(); let mut config = Config::default(); - config.exclude = flags.exclude; config.llvm_enabled = true; config.llvm_optimize = true; config.llvm_version_check = true; diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e7aed7eb4fead..460fb016f16ea 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1233,6 +1233,31 @@ impl Step for Rustfmt { } } + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct DontDistWithMiriEnabled; + +impl Step for DontDistWithMiriEnabled { + type Output = PathBuf; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun) -> ShouldRun { + let build_miri = run.builder.build.config.test_miri; + run.default_condition(build_miri) + } + + fn make_run(run: RunConfig) { + run.builder.ensure(DontDistWithMiriEnabled); + } + + fn run(self, _: &Builder) -> PathBuf { + panic!("Do not distribute with miri enabled.\n\ + The distributed libraries would include all MIR (increasing binary size). + The distributed MIR would include validation statements."); + } +} + + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Extended { stage: u32, diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 55d9723527e6d..6a75fc5112f5c 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -429,7 +429,7 @@ impl Step for Std { fn should_run(run: ShouldRun) -> ShouldRun { let builder = run.builder; - run.all_krates("std").default_condition(builder.build.config.docs) + run.krate("std").default_condition(builder.build.config.docs) } fn make_run(run: RunConfig) { diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 8a38fedc6136d..478e496078add 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -42,7 +42,6 @@ pub struct Flags { pub jobs: Option, pub cmd: Subcommand, pub incremental: bool, - pub exclude: Vec, } pub enum Subcommand { @@ -110,7 +109,6 @@ To learn more about a subcommand, run `./x.py -h`"); opts.optopt("", "build", "build target of the stage0 compiler", "BUILD"); opts.optmulti("", "host", "host targets to build", "HOST"); opts.optmulti("", "target", "target targets to build", "TARGET"); - opts.optmulti("", "exclude", "build paths to exclude", "PATH"); opts.optopt("", "on-fail", "command to run on failure", "CMD"); opts.optopt("", "stage", "stage to build", "N"); opts.optopt("", "keep-stage", "stage to keep without recompiling", "N"); @@ -275,12 +273,7 @@ Arguments: }; // Get any optional paths which occur after the subcommand let cwd = t!(env::current_dir()); - let src = matches.opt_str("src").map(PathBuf::from) - .or_else(|| env::var_os("SRC").map(PathBuf::from)) - .unwrap_or(cwd.clone()); - let paths = matches.free[1..].iter().map(|p| { - cwd.join(p).strip_prefix(&src).expect("paths passed to be inside checkout").into() - }).collect::>(); + let paths = matches.free[1..].iter().map(|p| cwd.join(p)).collect::>(); let cfg_file = matches.opt_str("config").map(PathBuf::from).or_else(|| { if fs::metadata("config.toml").is_ok() { @@ -365,6 +358,11 @@ Arguments: stage = Some(1); } + let cwd = t!(env::current_dir()); + let src = matches.opt_str("src").map(PathBuf::from) + .or_else(|| env::var_os("SRC").map(PathBuf::from)) + .unwrap_or(cwd); + Flags { verbose: matches.opt_count("verbose"), stage, @@ -376,14 +374,10 @@ Arguments: target: split(matches.opt_strs("target")) .into_iter().map(|x| INTERNER.intern_string(x)).collect::>(), config: cfg_file, + src, jobs: matches.opt_str("jobs").map(|j| j.parse().unwrap()), cmd, incremental: matches.opt_present("incremental"), - exclude: split(matches.opt_strs("exclude")) - .into_iter().map(|p| { - cwd.join(p).strip_prefix(&src).expect("paths to be inside checkout").into() - }).collect::>(), - src, } } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index afd740ce54845..83c270865c0b7 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -113,8 +113,9 @@ //! More documentation can be found in each respective module below, and you can //! also check out the `src/bootstrap/README.md` file for more information. -//#![deny(warnings)] -#![feature(core_intrinsics)] +#![deny(warnings)] +#![allow(stable_features)] +#![feature(associated_consts)] #[macro_use] extern crate build_helper; @@ -266,18 +267,6 @@ struct Crate { bench_step: String, } -impl Crate { - fn is_local(&self, build: &Build) -> bool { - self.path.starts_with(&build.config.src) && - !self.path.to_string_lossy().ends_with("_shim") - } - - fn local_path(&self, build: &Build) -> PathBuf { - assert!(self.is_local(build)); - self.path.strip_prefix(&build.config.src).unwrap().into() - } -} - /// The various "modes" of invoking Cargo. /// /// These entries currently correspond to the various output directories of the @@ -960,18 +949,22 @@ impl Build { } } - fn in_tree_crates(&self, root: &str) -> Vec<&Crate> { + /// Get a list of crates from a root crate. + /// + /// Returns Vec<(crate, path to crate, is_root_crate)> + fn crates(&self, root: &str) -> Vec<(Interned, &Path)> { + let interned = INTERNER.intern_string(root.to_owned()); let mut ret = Vec::new(); - let mut list = vec![INTERNER.intern_str(root)]; + let mut list = vec![interned]; let mut visited = HashSet::new(); while let Some(krate) = list.pop() { let krate = &self.crates[&krate]; - if krate.is_local(self) { - ret.push(krate); - for dep in &krate.deps { - if visited.insert(dep) && dep != "build_helper" { - list.push(*dep); - } + // If we can't strip prefix, then out-of-tree path + let path = krate.path.strip_prefix(&self.src).unwrap_or(&krate.path); + ret.push((krate.name, path)); + for dep in &krate.deps { + if visited.insert(dep) && dep != "build_helper" { + list.push(*dep); } } } diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 29cd23bdbb197..5a4791833fe0d 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -51,7 +51,9 @@ impl Step for Llvm { } fn make_run(run: RunConfig) { - let emscripten = run.path.ends_with("llvm-emscripten"); + let emscripten = run.path.map(|p| { + p.ends_with("llvm-emscripten") + }).unwrap_or(false); run.builder.ensure(Llvm { target: run.target, emscripten, diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 64ede4f4ecc88..f6b95f0bf9744 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -13,6 +13,7 @@ //! This file implements the various regression test suites that we execute on //! our CI. +use std::collections::HashSet; use std::env; use std::ffi::OsString; use std::iter; @@ -25,7 +26,6 @@ use std::io::Read; use build_helper::{self, output}; use builder::{Kind, RunConfig, ShouldRun, Builder, Compiler, Step}; -use Crate as CargoCrate; use cache::{INTERNER, Interned}; use compile; use dist; @@ -550,213 +550,180 @@ fn testdir(build: &Build, host: Interned) -> PathBuf { build.out.join(host).join("test") } -macro_rules! default_test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { - test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); - } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct Test { + path: &'static str, + mode: &'static str, + suite: &'static str, } -macro_rules! host_test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { - test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); - } +static DEFAULT_COMPILETESTS: &[Test] = &[ + Test { path: "src/test/ui", mode: "ui", suite: "ui" }, + Test { path: "src/test/run-pass", mode: "run-pass", suite: "run-pass" }, + Test { path: "src/test/compile-fail", mode: "compile-fail", suite: "compile-fail" }, + Test { path: "src/test/parse-fail", mode: "parse-fail", suite: "parse-fail" }, + Test { path: "src/test/run-fail", mode: "run-fail", suite: "run-fail" }, + Test { + path: "src/test/run-pass-valgrind", + mode: "run-pass-valgrind", + suite: "run-pass-valgrind" + }, + Test { path: "src/test/mir-opt", mode: "mir-opt", suite: "mir-opt" }, + Test { path: "src/test/codegen", mode: "codegen", suite: "codegen" }, + Test { path: "src/test/codegen-units", mode: "codegen-units", suite: "codegen-units" }, + Test { path: "src/test/incremental", mode: "incremental", suite: "incremental" }, + + // What this runs varies depending on the native platform being apple + Test { path: "src/test/debuginfo", mode: "debuginfo-XXX", suite: "debuginfo" }, +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct DefaultCompiletest { + compiler: Compiler, + target: Interned, + mode: &'static str, + suite: &'static str, } -macro_rules! test { - ($name:ident { - path: $path:expr, - mode: $mode:expr, - suite: $suite:expr, - default: $default:expr, - host: $host:expr - }) => { - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] - pub struct $name { - pub compiler: Compiler, - pub target: Interned, - } +impl Step for DefaultCompiletest { + type Output = (); + const DEFAULT: bool = true; - impl Step for $name { - type Output = (); - const DEFAULT: bool = $default; - const ONLY_HOSTS: bool = $host; + fn should_run(mut run: ShouldRun) -> ShouldRun { + for test in DEFAULT_COMPILETESTS { + run = run.path(test.path); + } + run + } - fn should_run(run: ShouldRun) -> ShouldRun { - run.path($path) - } + fn make_run(run: RunConfig) { + let compiler = run.builder.compiler(run.builder.top_stage, run.host); - fn make_run(run: RunConfig) { - let compiler = run.builder.compiler(run.builder.top_stage, run.host); + let test = run.path.map(|path| { + DEFAULT_COMPILETESTS.iter().find(|&&test| { + path.ends_with(test.path) + }).unwrap_or_else(|| { + panic!("make_run in compile test to receive test path, received {:?}", path); + }) + }); - run.builder.ensure($name { + if let Some(test) = test { + run.builder.ensure(DefaultCompiletest { + compiler, + target: run.target, + mode: test.mode, + suite: test.suite, + }); + } else { + for test in DEFAULT_COMPILETESTS { + run.builder.ensure(DefaultCompiletest { compiler, target: run.target, + mode: test.mode, + suite: test.suite }); } + } + } + + fn run(self, builder: &Builder) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: self.mode, + suite: self.suite, + }) + } +} - fn run(self, builder: &Builder) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: $mode, - suite: $suite, - }) +// Also default, but host-only. +static HOST_COMPILETESTS: &[Test] = &[ + Test { path: "src/test/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }, + Test { path: "src/test/run-pass-fulldeps", mode: "run-pass", suite: "run-pass-fulldeps" }, + Test { path: "src/test/run-fail-fulldeps", mode: "run-fail", suite: "run-fail-fulldeps" }, + Test { + path: "src/test/compile-fail-fulldeps", + mode: "compile-fail", + suite: "compile-fail-fulldeps", + }, + Test { + path: "src/test/incremental-fulldeps", + mode: "incremental", + suite: "incremental-fulldeps", + }, + Test { path: "src/test/rustdoc", mode: "rustdoc", suite: "rustdoc" }, + + Test { path: "src/test/pretty", mode: "pretty", suite: "pretty" }, + Test { path: "src/test/run-pass/pretty", mode: "pretty", suite: "run-pass" }, + Test { path: "src/test/run-fail/pretty", mode: "pretty", suite: "run-fail" }, + Test { path: "src/test/run-pass-valgrind/pretty", mode: "pretty", suite: "run-pass-valgrind" }, + Test { path: "src/test/run-pass-fulldeps/pretty", mode: "pretty", suite: "run-pass-fulldeps" }, + Test { path: "src/test/run-fail-fulldeps/pretty", mode: "pretty", suite: "run-fail-fulldeps" }, + Test { path: "src/test/run-make", mode: "run-make", suite: "run-make" }, +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct HostCompiletest { + compiler: Compiler, + target: Interned, + mode: &'static str, + suite: &'static str, +} + +impl Step for HostCompiletest { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(mut run: ShouldRun) -> ShouldRun { + for test in HOST_COMPILETESTS { + run = run.path(test.path); + } + run + } + + fn make_run(run: RunConfig) { + let compiler = run.builder.compiler(run.builder.top_stage, run.host); + + let test = run.path.map(|path| { + HOST_COMPILETESTS.iter().find(|&&test| { + path.ends_with(test.path) + }).unwrap_or_else(|| { + panic!("make_run in compile test to receive test path, received {:?}", path); + }) + }); + + if let Some(test) = test { + run.builder.ensure(HostCompiletest { + compiler, + target: run.target, + mode: test.mode, + suite: test.suite, + }); + } else { + for test in HOST_COMPILETESTS { + if test.mode == "pretty" { + continue; + } + run.builder.ensure(HostCompiletest { + compiler, + target: run.target, + mode: test.mode, + suite: test.suite + }); } } } -} -default_test!(Ui { - path: "src/test/ui", - mode: "ui", - suite: "ui" -}); - -default_test!(RunPass { - path: "src/test/run-pass", - mode: "run-pass", - suite: "run-pass" -}); - -default_test!(CompileFail { - path: "src/test/compile-fail", - mode: "compile-fail", - suite: "compile-fail" -}); - -default_test!(ParseFail { - path: "src/test/parse-fail", - mode: "parse-fail", - suite: "parse-fail" -}); - -default_test!(RunFail { - path: "src/test/run-fail", - mode: "run-fail", - suite: "run-fail" -}); - -default_test!(RunPassValgrind { - path: "src/test/run-pass-valgrind", - mode: "run-pass-valgrind", - suite: "run-pass-valgrind" -}); - -default_test!(MirOpt { - path: "src/test/mir-opt", - mode: "mir-opt", - suite: "mir-opt" -}); - -default_test!(Codegen { - path: "src/test/codegen", - mode: "codegen", - suite: "codegen" -}); - -default_test!(CodegenUnits { - path: "src/test/codegen-units", - mode: "codegen-units", - suite: "codegen-units" -}); - -default_test!(Incremental { - path: "src/test/incremental", - mode: "incremental", - suite: "incremental" -}); - -default_test!(Debuginfo { - path: "src/test/debuginfo", - // What this runs varies depending on the native platform being apple - mode: "debuginfo-XXX", - suite: "debuginfo" -}); - -host_test!(UiFullDeps { - path: "src/test/ui-fulldeps", - mode: "ui", - suite: "ui-fulldeps" -}); - -host_test!(RunPassFullDeps { - path: "src/test/run-pass-fulldeps", - mode: "run-pass", - suite: "run-pass-fulldeps" -}); - -host_test!(RunFailFullDeps { - path: "src/test/run-fail-fulldeps", - mode: "run-fail", - suite: "run-fail-fulldeps" -}); - -host_test!(CompileFailFullDeps { - path: "src/test/compile-fail-fulldeps", - mode: "compile-fail", - suite: "compile-fail-fulldeps" -}); - -host_test!(IncrementalFullDeps { - path: "src/test/incremental-fulldeps", - mode: "incremental", - suite: "incremental-fulldeps" -}); - -host_test!(Rustdoc { - path: "src/test/rustdoc", - mode: "rustdoc", - suite: "rustdoc" -}); - -test!(Pretty { - path: "src/test/pretty", - mode: "pretty", - suite: "pretty", - default: false, - host: true -}); -test!(RunPassPretty { - path: "src/test/run-pass/pretty", - mode: "pretty", - suite: "run-pass", - default: false, - host: true -}); -test!(RunFailPretty { - path: "src/test/run-fail/pretty", - mode: "pretty", - suite: "run-fail", - default: false, - host: true -}); -test!(RunPassValgrindPretty { - path: "src/test/run-pass-valgrind/pretty", - mode: "pretty", - suite: "run-pass-valgrind", - default: false, - host: true -}); -test!(RunPassFullDepsPretty { - path: "src/test/run-pass-fulldeps/pretty", - mode: "pretty", - suite: "run-pass-fulldeps", - default: false, - host: true -}); -test!(RunFailFullDepsPretty { - path: "src/test/run-fail-fulldeps/pretty", - mode: "pretty", - suite: "run-fail-fulldeps", - default: false, - host: true -}); - -host_test!(RunMake { - path: "src/test/run-make", - mode: "run-make", - suite: "run-make" -}); + fn run(self, builder: &Builder) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: self.mode, + suite: self.suite, + }) + } +} #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] struct Compiletest { @@ -935,7 +902,7 @@ impl Step for Compiletest { } } if suite == "run-make" && !build.config.llvm_enabled { - println!("Ignoring run-make test suite as they generally dont work without LLVM"); + println!("Ignoring run-make test suite as they generally don't work without LLVM"); return; } @@ -1132,7 +1099,7 @@ pub struct CrateLibrustc { compiler: Compiler, target: Interned, test_kind: TestKind, - krate: Interned, + krate: Option>, } impl Step for CrateLibrustc { @@ -1148,26 +1115,35 @@ impl Step for CrateLibrustc { let builder = run.builder; let compiler = builder.compiler(builder.top_stage, run.host); - for krate in builder.in_tree_crates("rustc-main") { - if run.path.ends_with(&krate.path) { - let test_kind = if builder.kind == Kind::Test { - TestKind::Test - } else if builder.kind == Kind::Bench { - TestKind::Bench - } else { - panic!("unexpected builder.kind in crate: {:?}", builder.kind); - }; + let make = |name: Option>| { + let test_kind = if builder.kind == Kind::Test { + TestKind::Test + } else if builder.kind == Kind::Bench { + TestKind::Bench + } else { + panic!("unexpected builder.kind in crate: {:?}", builder.kind); + }; - builder.ensure(CrateLibrustc { - compiler, - target: run.target, - test_kind, - krate: krate.name, - }); + builder.ensure(CrateLibrustc { + compiler, + target: run.target, + test_kind, + krate: name, + }); + }; + + if let Some(path) = run.path { + for (name, krate_path) in builder.crates("rustc-main") { + if path.ends_with(krate_path) { + make(Some(name)); + } } + } else { + make(None); } } + fn run(self, builder: &Builder) { builder.ensure(Crate { compiler: self.compiler, @@ -1179,96 +1155,28 @@ impl Step for CrateLibrustc { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrateNotDefault { - compiler: Compiler, - target: Interned, - test_kind: TestKind, - krate: &'static str, -} - -impl Step for CrateNotDefault { - type Output = (); - - fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/liballoc_jemalloc") - .path("src/librustc_asan") - .path("src/librustc_lsan") - .path("src/librustc_msan") - .path("src/librustc_tsan") - } - - fn make_run(run: RunConfig) { - let builder = run.builder; - let compiler = builder.compiler(builder.top_stage, run.host); - - let test_kind = if builder.kind == Kind::Test { - TestKind::Test - } else if builder.kind == Kind::Bench { - TestKind::Bench - } else { - panic!("unexpected builder.kind in crate: {:?}", builder.kind); - }; - - builder.ensure(CrateNotDefault { - compiler, - target: run.target, - test_kind, - krate: match run.path { - _ if run.path.ends_with("src/liballoc_jemalloc") => "alloc_jemalloc", - _ if run.path.ends_with("src/librustc_asan") => "rustc_asan", - _ if run.path.ends_with("src/librustc_lsan") => "rustc_lsan", - _ if run.path.ends_with("src/librustc_msan") => "rustc_msan", - _ if run.path.ends_with("src/librustc_tsan") => "rustc_tsan", - _ => panic!("unexpected path {:?}", run.path), - }, - }); - } - - fn run(self, builder: &Builder) { - builder.ensure(Crate { - compiler: self.compiler, - target: self.target, - mode: Mode::Libstd, - test_kind: self.test_kind, - krate: INTERNER.intern_str(self.krate), - }); - } -} - - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct Crate { compiler: Compiler, target: Interned, mode: Mode, test_kind: TestKind, - krate: Interned, + krate: Option>, } impl Step for Crate { type Output = (); const DEFAULT: bool = true; - fn should_run(mut run: ShouldRun) -> ShouldRun { - let builder = run.builder; - run = run.krate("test"); - for krate in run.builder.in_tree_crates("std") { - if krate.is_local(&run.builder) && - !krate.name.contains("jemalloc") && - !(krate.name.starts_with("rustc_") && krate.name.ends_with("san")) && - krate.name != "dlmalloc" { - run = run.path(krate.local_path(&builder).to_str().unwrap()); - } - } - run + fn should_run(run: ShouldRun) -> ShouldRun { + run.krate("std").krate("test") } fn make_run(run: RunConfig) { let builder = run.builder; let compiler = builder.compiler(builder.top_stage, run.host); - let make = |mode: Mode, krate: &CargoCrate| { + let make = |mode: Mode, name: Option>| { let test_kind = if builder.kind == Kind::Test { TestKind::Test } else if builder.kind == Kind::Bench { @@ -1282,24 +1190,29 @@ impl Step for Crate { target: run.target, mode, test_kind, - krate: krate.name, + krate: name, }); }; - for krate in builder.in_tree_crates("std") { - if run.path.ends_with(&krate.local_path(&builder)) { - make(Mode::Libstd, krate); + if let Some(path) = run.path { + for (name, krate_path) in builder.crates("std") { + if path.ends_with(krate_path) { + make(Mode::Libstd, Some(name)); + } } - } - for krate in builder.in_tree_crates("test") { - if run.path.ends_with(&krate.local_path(&builder)) { - make(Mode::Libtest, krate); + for (name, krate_path) in builder.crates("test") { + if path.ends_with(krate_path) { + make(Mode::Libtest, Some(name)); + } } + } else { + make(Mode::Libstd, None); + make(Mode::Libtest, None); } } - /// Run all unit tests plus documentation tests for a given crate defined - /// by a `Cargo.toml` (single manifest) + /// Run all unit tests plus documentation tests for an entire crate DAG defined + /// by a `Cargo.toml` /// /// This is what runs tests for crates like the standard library, compiler, etc. /// It essentially is the driver for running `cargo test`. @@ -1328,23 +1241,27 @@ impl Step for Crate { }; let mut cargo = builder.cargo(compiler, mode, target, test_kind.subcommand()); - match mode { + let (name, root) = match mode { Mode::Libstd => { compile::std_cargo(build, &compiler, target, &mut cargo); + ("libstd", "std") } Mode::Libtest => { compile::test_cargo(build, &compiler, target, &mut cargo); + ("libtest", "test") } Mode::Librustc => { builder.ensure(compile::Rustc { compiler, target }); compile::rustc_cargo(build, &mut cargo); + ("librustc", "rustc-main") } _ => panic!("can only test libraries"), }; + let root = INTERNER.intern_string(String::from(root)); let _folder = build.fold_output(|| { - format!("{}_stage{}-{}", test_kind.subcommand(), compiler.stage, krate) + format!("{}_stage{}-{}", test_kind.subcommand(), compiler.stage, name) }); - println!("{} {} stage{} ({} -> {})", test_kind, krate, compiler.stage, + println!("{} {} stage{} ({} -> {})", test_kind, name, compiler.stage, &compiler.host, target); // Build up the base `cargo test` command. @@ -1356,7 +1273,37 @@ impl Step for Crate { cargo.arg("--no-fail-fast"); } - cargo.arg("-p").arg(krate); + match krate { + Some(krate) => { + cargo.arg("-p").arg(krate); + } + None => { + let mut visited = HashSet::new(); + let mut next = vec![root]; + while let Some(name) = next.pop() { + // Right now jemalloc and the sanitizer crates are + // target-specific crate in the sense that it's not present + // on all platforms. Custom skip it here for now, but if we + // add more this probably wants to get more generalized. + // + // Also skip `build_helper` as it's not compiled normally + // for target during the bootstrap and it's just meant to be + // a helper crate, not tested. If it leaks through then it + // ends up messing with various mtime calculations and such. + if !name.contains("jemalloc") && + *name != *"build_helper" && + !(name.starts_with("rustc_") && name.ends_with("san")) && + name != "dlmalloc" { + cargo.arg("-p").arg(&format!("{}:0.0.0", name)); + } + for dep in build.crates[&name].deps.iter() { + if visited.insert(dep) { + next.push(*dep); + } + } + } + } + } // The tests are going to run with the *target* libraries, so we need to // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. @@ -1408,18 +1355,18 @@ impl Step for Crate { } #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrateRustdoc { +pub struct Rustdoc { host: Interned, test_kind: TestKind, } -impl Step for CrateRustdoc { +impl Step for Rustdoc { type Output = (); const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.paths(&["src/librustdoc", "src/tools/rustdoc"]) + run.path("src/librustdoc").path("src/tools/rustdoc") } fn make_run(run: RunConfig) { @@ -1433,7 +1380,7 @@ impl Step for CrateRustdoc { panic!("unexpected builder.kind in crate: {:?}", builder.kind); }; - builder.ensure(CrateRustdoc { + builder.ensure(Rustdoc { host: run.host, test_kind, });