diff --git a/config.toml.example b/config.toml.example index df4478bb0cb85..932600ac3aaee 100644 --- a/config.toml.example +++ b/config.toml.example @@ -798,9 +798,6 @@ changelog-seen = 2 # on linux #src-tarball = true -# Whether to allow failures when building tools -#missing-tools = false - # List of compression formats to use when generating dist tarballs. The list of # formats is provided to rust-installer, which must support all of them. # diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 56f96734bbbc7..5fcb0b8d81f1a 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -202,7 +202,6 @@ pub struct Config { pub save_toolstates: Option, pub print_step_timings: bool, pub print_step_rusage: bool, - pub missing_tools: bool, // Fallback musl-root for all targets pub musl_root: Option, @@ -700,7 +699,6 @@ define_config! { gpg_password_file: Option = "gpg-password-file", upload_addr: Option = "upload-addr", src_tarball: Option = "src-tarball", - missing_tools: Option = "missing-tools", compression_formats: Option> = "compression-formats", } } @@ -1306,7 +1304,6 @@ impl Config { config.dist_upload_addr = t.upload_addr; config.dist_compression_formats = t.compression_formats; set(&mut config.rust_dist_src, t.src_tarball); - set(&mut config.missing_tools, t.missing_tools); } if let Some(r) = build.rustfmt { diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 0af329e7007ac..38cf4ae8ac5c1 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -57,7 +57,6 @@ def v(*args): o("full-tools", None, "enable all tools") o("lld", "rust.lld", "build lld") o("clang", "llvm.clang", "build clang") -o("missing-tools", "dist.missing-tools", "allow failures when building tools") o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9b2b549612d81..f4d50ff765bbe 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1103,9 +1103,7 @@ impl Step for Rls { let compiler = self.compiler; let target = self.target; - let rls = builder - .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) - .expect("rls expected to build"); + let rls = builder.ensure(tool::Rls { compiler, target, extra_features: Vec::new() }); let mut tarball = Tarball::new(builder, "rls", &target.triple); tarball.set_overlay(OverlayKind::RLS); @@ -1147,10 +1145,7 @@ impl Step for RustAnalyzer { let compiler = self.compiler; let target = self.target; - let rust_analyzer = builder - .ensure(tool::RustAnalyzer { compiler, target }) - .expect("rust-analyzer always builds"); - + let rust_analyzer = builder.ensure(tool::RustAnalyzer { compiler, target }); let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); tarball.set_overlay(OverlayKind::RustAnalyzer); tarball.is_preview(true); @@ -1194,12 +1189,9 @@ impl Step for Clippy { // Prepare the image directory // We expect clippy to build, because we've exited this step above if tool // state for clippy isn't testing. - let clippy = builder - .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }) - .expect("clippy expected to build - essential tool"); - let cargoclippy = builder - .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) - .expect("clippy expected to build - essential tool"); + let clippy = builder.ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }); + let cargoclippy = + builder.ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }); let mut tarball = Tarball::new(builder, "clippy", &target.triple); tarball.set_overlay(OverlayKind::Clippy); @@ -1248,9 +1240,9 @@ impl Step for Miri { let compiler = self.compiler; let target = self.target; - let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?; + let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() }); let cargomiri = - builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?; + builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() }); let mut tarball = Tarball::new(builder, "miri", &target.triple); tarball.set_overlay(OverlayKind::Miri); @@ -1293,12 +1285,10 @@ impl Step for Rustfmt { let compiler = self.compiler; let target = self.target; - let rustfmt = builder - .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) - .expect("rustfmt expected to build - essential tool"); - let cargofmt = builder - .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }) - .expect("cargo fmt expected to build - essential tool"); + let rustfmt = + builder.ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }); + let cargofmt = + builder.ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }); let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); tarball.set_overlay(OverlayKind::Rustfmt); tarball.is_preview(true); @@ -1352,9 +1342,8 @@ impl Step for RustDemangler { return None; } - let rust_demangler = builder - .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) - .expect("rust-demangler expected to build - in-tree tool"); + let rust_demangler = + builder.ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }); // Prepare the image directory let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index f753720b35306..750d387c9ab59 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -227,8 +227,6 @@ pub struct Build { ci_env: CiEnv, delayed_failures: RefCell>, prerelease_version: Cell>, - tool_artifacts: - RefCell)>>>, #[cfg(feature = "build-metrics")] metrics: metrics::BuildMetrics, @@ -447,7 +445,6 @@ impl Build { ci_env: CiEnv::current(), delayed_failures: RefCell::new(Vec::new()), prerelease_version: Cell::new(None), - tool_artifacts: Default::default(), #[cfg(feature = "build-metrics")] metrics: metrics::BuildMetrics::init(), diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index e14440f57a8a6..3b4623070e560 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -161,9 +161,8 @@ impl Step for Miri { let target = self.target; let compiler = builder.compiler(stage, host); - let miri = builder - .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); + let miri = + builder.ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }); let miri_sysroot = test::Miri::build_miri_sysroot(builder, compiler, &miri, target); // # Run miri. diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 9cd6107b43ac3..e85955de2a22c 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -362,7 +362,7 @@ impl Step for RustAnalyzer { let host = self.host; let compiler = builder.compiler(stage, host); - builder.ensure(tool::RustAnalyzer { compiler, target: self.host }).expect("in-tree tool"); + builder.ensure(tool::RustAnalyzer { compiler, target: self.host }); let workspace_path = "src/tools/rust-analyzer"; // until the whole RA test suite runs on `i686`, we only run @@ -420,9 +420,7 @@ impl Step for Rustfmt { let host = self.host; let compiler = builder.compiler(stage, host); - builder - .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); + builder.ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }); let mut cargo = tool::prepare_tool_cargo( builder, @@ -469,9 +467,11 @@ impl Step for RustDemangler { let host = self.host; let compiler = builder.compiler(stage, host); - let rust_demangler = builder - .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); + let rust_demangler = builder.ensure(tool::RustDemangler { + compiler, + target: self.host, + extra_features: Vec::new(), + }); let mut cargo = tool::prepare_tool_cargo( builder, compiler, @@ -589,12 +589,13 @@ impl Step for Miri { // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage. let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host); - let miri = builder - .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let _cargo_miri = builder - .ensure(tool::CargoMiri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); + let miri = + builder.ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }); + let _cargo_miri = builder.ensure(tool::CargoMiri { + compiler, + target: self.host, + extra_features: Vec::new(), + }); // The stdlib we need might be at a different stage. And just asking for the // sysroot does not seem to populate it, so we do that first. builder.ensure(compile::Std::new(compiler_std, host)); @@ -732,9 +733,7 @@ impl Step for Clippy { let host = self.host; let compiler = builder.compiler(stage, host); - builder - .ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); + builder.ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }); let mut cargo = tool::prepare_tool_cargo( builder, compiler, @@ -1458,13 +1457,11 @@ note: if you're sure you want to do this, please open an issue as to why. In the } if mode == "run-make" { - let rust_demangler = builder - .ensure(tool::RustDemangler { - compiler, - target: compiler.host, - extra_features: Vec::new(), - }) - .expect("in-tree tool"); + let rust_demangler = builder.ensure(tool::RustDemangler { + compiler, + target: compiler.host, + extra_features: Vec::new(), + }); cmd.arg("--rust-demangler-path").arg(rust_demangler); } diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 3c9a154da9ab1..99ffdcfe26ddb 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -1,4 +1,3 @@ -use std::collections::HashSet; use std::env; use std::fs; use std::path::PathBuf; @@ -26,7 +25,6 @@ struct ToolBuild { tool: &'static str, path: &'static str, mode: Mode, - is_optional_tool: bool, source_type: SourceType, extra_features: Vec, /// Nightly-only features that are allowed (comma-separated list). @@ -72,7 +70,7 @@ fn tooling_output( } impl Step for ToolBuild { - type Output = Option; + type Output = PathBuf; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.never() @@ -82,12 +80,11 @@ impl Step for ToolBuild { /// /// This will build the specified tool with the specified `host` compiler in /// `stage` into the normal cargo output directory. - fn run(self, builder: &Builder<'_>) -> Option { + fn run(self, builder: &Builder<'_>) -> PathBuf { let compiler = self.compiler; let target = self.target; let mut tool = self.tool; let path = self.path; - let is_optional_tool = self.is_optional_tool; match self.mode { Mode::ToolRustc => { @@ -120,159 +117,24 @@ impl Step for ToolBuild { &self.target, ); builder.info(&msg); - let mut duplicates = Vec::new(); - let is_expected = compile::stream_cargo(builder, cargo, vec![], &mut |msg| { - // Only care about big things like the RLS/Cargo for now - match tool { - "rls" | "cargo" | "clippy-driver" | "miri" | "rustfmt" => {} - - _ => return, - } - let (id, features, filenames) = match msg { - compile::CargoMessage::CompilerArtifact { - package_id, - features, - filenames, - target: _, - } => (package_id, features, filenames), - _ => return, - }; - let features = features.iter().map(|s| s.to_string()).collect::>(); - - for path in filenames { - let val = (tool, PathBuf::from(&*path), features.clone()); - // we're only interested in deduplicating rlibs for now - if val.1.extension().and_then(|s| s.to_str()) != Some("rlib") { - continue; - } - - // Don't worry about compiles that turn out to be host - // dependencies or build scripts. To skip these we look for - // anything that goes in `.../release/deps` but *doesn't* go in - // `$target/release/deps`. This ensure that outputs in - // `$target/release` are still considered candidates for - // deduplication. - if let Some(parent) = val.1.parent() { - if parent.ends_with("release/deps") { - let maybe_target = parent - .parent() - .and_then(|p| p.parent()) - .and_then(|p| p.file_name()) - .and_then(|p| p.to_str()) - .unwrap(); - if maybe_target != &*target.triple { - continue; - } - } - } - - // Record that we've built an artifact for `id`, and if one was - // already listed then we need to see if we reused the same - // artifact or produced a duplicate. - let mut artifacts = builder.tool_artifacts.borrow_mut(); - let prev_artifacts = artifacts.entry(target).or_default(); - let prev = match prev_artifacts.get(&*id) { - Some(prev) => prev, - None => { - prev_artifacts.insert(id.to_string(), val); - continue; - } - }; - if prev.1 == val.1 { - return; // same path, same artifact - } - - // If the paths are different and one of them *isn't* inside of - // `release/deps`, then it means it's probably in - // `$target/release`, or it's some final artifact like - // `libcargo.rlib`. In these situations Cargo probably just - // copied it up from `$target/release/deps/libcargo-xxxx.rlib`, - // so if the features are equal we can just skip it. - let prev_no_hash = prev.1.parent().unwrap().ends_with("release/deps"); - let val_no_hash = val.1.parent().unwrap().ends_with("release/deps"); - if prev.2 == val.2 || !prev_no_hash || !val_no_hash { - return; - } - - // ... and otherwise this looks like we duplicated some sort of - // compilation, so record it to generate an error later. - duplicates.push((id.to_string(), val, prev.clone())); - } - }); + builder.save_toolstate(tool, ToolState::TestFail); - if is_expected && !duplicates.is_empty() { - eprintln!( - "duplicate artifacts found when compiling a tool, this \ - typically means that something was recompiled because \ - a transitive dependency has different features activated \ - than in a previous build:\n" - ); - let (same, different): (Vec<_>, Vec<_>) = - duplicates.into_iter().partition(|(_, cur, prev)| cur.2 == prev.2); - if !same.is_empty() { - eprintln!( - "the following dependencies are duplicated although they \ - have the same features enabled:" - ); - for (id, cur, prev) in same { - eprintln!(" {}", id); - // same features - eprintln!(" `{}` ({:?})\n `{}` ({:?})", cur.0, cur.1, prev.0, prev.1); - } - } - if !different.is_empty() { - eprintln!("the following dependencies have different features:"); - for (id, cur, prev) in different { - eprintln!(" {}", id); - let cur_features: HashSet<_> = cur.2.into_iter().collect(); - let prev_features: HashSet<_> = prev.2.into_iter().collect(); - eprintln!( - " `{}` additionally enabled features {:?} at {:?}", - cur.0, - &cur_features - &prev_features, - cur.1 - ); - eprintln!( - " `{}` additionally enabled features {:?} at {:?}", - prev.0, - &prev_features - &cur_features, - prev.1 - ); - } - } - eprintln!(); - eprintln!( - "to fix this you will probably want to edit the local \ - src/tools/rustc-workspace-hack/Cargo.toml crate, as \ - that will update the dependency graph to ensure that \ - these crates all share the same feature set" - ); - panic!("tools should not compile multiple copies of the same crate"); + // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and + // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something + // different so the problem doesn't come up. + if tool == "tidy" { + tool = "rust-tidy"; } - builder.save_toolstate( - tool, - if is_expected { ToolState::TestFail } else { ToolState::BuildFail }, - ); + let out_dir = builder.cargo_out(compiler, self.mode, target); + let cargo_out = out_dir.join(exe(tool, target)); + let bootstrap_tool_stamp = out_dir.join(".bootstrap-tool.stamp"); + let bin = builder.tools_dir(compiler).join(exe(tool, target)); - if !is_expected { - if !is_optional_tool { - crate::detail_exit(1); - } else { - None - } - } else { - // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and - // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something - // different so the problem doesn't come up. - if tool == "tidy" { - tool = "rust-tidy"; - } - let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target)); - let bin = builder.tools_dir(compiler).join(exe(tool, target)); - builder.copy(&cargo_out, &bin); - Some(bin) - } + // Streamlined building all tools mandatorily + compile::run_cargo(builder, cargo, vec![], &bootstrap_tool_stamp, vec![], false, false); + builder.copy(&cargo_out, &bin); + bin } } @@ -398,7 +260,6 @@ macro_rules! bootstrap_tool { Mode::ToolBootstrap }, path: $path, - is_optional_tool: false, source_type: if false $(|| $external)* { SourceType::Submodule } else { @@ -406,7 +267,7 @@ macro_rules! bootstrap_tool { }, extra_features: vec![], allow_features: concat!($($allow_features)*), - }).expect("expected to build -- essential tool") + }) } } )+ @@ -476,19 +337,16 @@ impl Step for ErrorIndex { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.compiler.host, - tool: "error_index_generator", - mode: Mode::ToolRustc, - path: "src/tools/error_index_generator", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool") + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.compiler.host, + tool: "error_index_generator", + mode: Mode::ToolRustc, + path: "src/tools/error_index_generator", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) } } @@ -513,19 +371,16 @@ impl Step for RemoteTestServer { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "remote-test-server", - mode: Mode::ToolStd, - path: "src/tools/remote-test-server", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool") + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "remote-test-server", + mode: Mode::ToolStd, + path: "src/tools/remote-test-server", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) } } @@ -673,19 +528,16 @@ impl Step for Cargo { } fn run(self, builder: &Builder<'_>) -> PathBuf { - let cargo_bin_path = builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "cargo", - mode: Mode::ToolRustc, - path: "src/tools/cargo", - is_optional_tool: false, - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool"); + let cargo_bin_path = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "cargo", + mode: Mode::ToolRustc, + path: "src/tools/cargo", + source_type: SourceType::Submodule, + extra_features: Vec::new(), + allow_features: "", + }); let build_cred = |name, path| { // These credential helpers are currently experimental. @@ -696,7 +548,6 @@ impl Step for Cargo { tool: name, mode: Mode::ToolRustc, path, - is_optional_tool: true, source_type: SourceType::Submodule, extra_features: Vec::new(), allow_features: "", @@ -737,19 +588,16 @@ impl Step for LldWrapper { } fn run(self, builder: &Builder<'_>) -> PathBuf { - let src_exe = builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "lld-wrapper", - mode: Mode::ToolStd, - path: "src/tools/lld-wrapper", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool"); + let src_exe = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "lld-wrapper", + mode: Mode::ToolStd, + path: "src/tools/lld-wrapper", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }); src_exe } @@ -767,7 +615,7 @@ impl RustAnalyzer { } impl Step for RustAnalyzer { - type Output = Option; + type Output = PathBuf; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -790,7 +638,7 @@ impl Step for RustAnalyzer { }); } - fn run(self, builder: &Builder<'_>) -> Option { + fn run(self, builder: &Builder<'_>) -> PathBuf { builder.ensure(ToolBuild { compiler: self.compiler, target: self.target, @@ -798,7 +646,6 @@ impl Step for RustAnalyzer { mode: Mode::ToolStd, path: "src/tools/rust-analyzer", extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()], - is_optional_tool: false, source_type: SourceType::InTree, allow_features: RustAnalyzer::ALLOW_FEATURES, }) @@ -843,10 +690,9 @@ impl Step for RustAnalyzerProcMacroSrv { mode: Mode::ToolStd, path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", extra_features: vec!["proc-macro-srv/sysroot-abi".to_owned()], - is_optional_tool: false, source_type: SourceType::InTree, allow_features: RustAnalyzer::ALLOW_FEATURES, - })?; + }); // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` // so that r-a can use it. @@ -876,7 +722,7 @@ macro_rules! tool_extended { } impl Step for $name { - type Output = Option; + type Output = PathBuf; const DEFAULT: bool = true; // Overwritten below const ONLY_HOSTS: bool = true; @@ -907,7 +753,7 @@ macro_rules! tool_extended { } #[allow(unused_mut)] - fn run(mut $sel, $builder: &Builder<'_>) -> Option { + fn run(mut $sel, $builder: &Builder<'_>) -> PathBuf { $builder.ensure(ToolBuild { compiler: $sel.compiler, target: $sel.target, @@ -915,7 +761,6 @@ macro_rules! tool_extended { mode: if false $(|| $tool_std)? { Mode::ToolStd } else { Mode::ToolRustc }, path: $path, extra_features: $sel.extra_features, - is_optional_tool: true, source_type: SourceType::InTree, allow_features: concat!($($allow_features)*), }) diff --git a/src/ci/run.sh b/src/ci/run.sh index 93dccb54c4e38..6b0a4d3cbfcf9 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -125,10 +125,6 @@ else fi fi -if [ "$RUST_RELEASE_CHANNEL" = "nightly" ] || [ "$DIST_REQUIRE_ALL_TOOLS" = "" ]; then - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-missing-tools" -fi - export COMPILETEST_NEEDS_ALL_LLVM_COMPONENTS=1 # Print the date from the local machine and the date from an external source to