From 51d27a63b8039765790c469ebb169802da4a307e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 20 Mar 2024 13:52:36 +0100 Subject: [PATCH 1/7] Move cleanup of sysroot build into its own function --- build_system/src/build.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index c81b02e2183d8..2d66fd89fb9f9 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -55,8 +55,7 @@ impl BuildArg { } } -pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Result<(), String> { - let start_dir = Path::new("build_sysroot"); +fn cleanup_sysroot_previous_build(start_dir: &Path) { // Cleanup for previous run // Clean target dir except for build scripts and incremental cache let _ = walk_dir( @@ -100,6 +99,11 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu let _ = fs::remove_file(start_dir.join("Cargo.lock")); let _ = fs::remove_file(start_dir.join("test_target/Cargo.lock")); let _ = fs::remove_dir_all(start_dir.join("sysroot")); +} + +pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Result<(), String> { + let start_dir = Path::new("build_sysroot"); + cleanup_sysroot_previous_build(&start_dir); // Builds libs let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); From 9b17b3d184a77c6022f51b68aefbad60d0a47d81 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 20 Mar 2024 14:18:25 +0100 Subject: [PATCH 2/7] Simplify directory creation --- build_system/src/build.rs | 26 +++++--------------------- build_system/src/config.rs | 10 ++-------- build_system/src/prepare.rs | 10 ++-------- build_system/src/test.rs | 10 ++++------ build_system/src/utils.rs | 6 ++++++ 5 files changed, 19 insertions(+), 43 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 2d66fd89fb9f9..a4ea5107a0505 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -1,5 +1,5 @@ use crate::config::{Channel, ConfigInfo}; -use crate::utils::{run_command, run_command_with_output_and_env, walk_dir}; +use crate::utils::{create_dir, run_command, run_command_with_output_and_env, walk_dir}; use std::collections::HashMap; use std::ffi::OsStr; use std::fs; @@ -103,6 +103,7 @@ fn cleanup_sysroot_previous_build(start_dir: &Path) { pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Result<(), String> { let start_dir = Path::new("build_sysroot"); + cleanup_sysroot_previous_build(&start_dir); // Builds libs @@ -136,13 +137,7 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu // Copy files to sysroot let sysroot_path = start_dir.join(format!("sysroot/lib/rustlib/{}/lib/", config.target_triple)); - fs::create_dir_all(&sysroot_path).map_err(|error| { - format!( - "Failed to create directory `{}`: {:?}", - sysroot_path.display(), - error - ) - })?; + create_dir(&sysroot_path)?; let copier = |dir_to_copy: &Path| { // FIXME: should not use shell command! run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ()) @@ -155,13 +150,7 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu // Copy the source files to the sysroot (Rust for Linux needs this). let sysroot_src_path = start_dir.join("sysroot/lib/rustlib/src/rust"); - fs::create_dir_all(&sysroot_src_path).map_err(|error| { - format!( - "Failed to create directory `{}`: {:?}", - sysroot_src_path.display(), - error - ) - })?; + create_dir(&sysroot_src_path)?; run_command( &[ &"cp", @@ -216,12 +205,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> { // We voluntarily ignore the error. let _ = fs::remove_dir_all("target/out"); let gccjit_target = "target/out/gccjit"; - fs::create_dir_all(gccjit_target).map_err(|error| { - format!( - "Failed to create directory `{}`: {:?}", - gccjit_target, error - ) - })?; + create_dir(gccjit_target)?; println!("[BUILD] sysroot"); build_sysroot(&env, &args.config_info)?; diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 4cda356301ada..2e6d1b038c108 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -1,5 +1,5 @@ use crate::utils::{ - create_symlink, get_os_name, run_command_with_output, rustc_version_info, split_args, + create_dir, create_symlink, get_os_name, run_command_with_output, rustc_version_info, split_args, }; use std::collections::HashMap; use std::env as std_env; @@ -228,13 +228,7 @@ impl ConfigInfo { let output_dir = output_dir.join(&commit); if !output_dir.is_dir() { - std::fs::create_dir_all(&output_dir).map_err(|err| { - format!( - "failed to create folder `{}`: {:?}", - output_dir.display(), - err, - ) - })?; + create_dir(&output_dir)?; } let output_dir = output_dir.canonicalize().map_err(|err| { format!( diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs index 821c793c7e554..fd577fa94341f 100644 --- a/build_system/src/prepare.rs +++ b/build_system/src/prepare.rs @@ -1,6 +1,6 @@ use crate::rustc_info::get_rustc_path; use crate::utils::{ - cargo_install, git_clone_root_dir, remove_file, run_command, run_command_with_output, walk_dir, + cargo_install, create_dir, git_clone_root_dir, remove_file, run_command, run_command_with_output, walk_dir, }; use std::fs; @@ -41,13 +41,7 @@ fn prepare_libcore( } let sysroot_library_dir = sysroot_dir.join("library"); - fs::create_dir_all(&sysroot_library_dir).map_err(|error| { - format!( - "Failed to create folder `{}`: {:?}", - sysroot_library_dir.display(), - error, - ) - })?; + create_dir(&sysroot_library_dir)?; run_command( &[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir], diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 0895dc6bff700..c0cb6c8f2397b 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1,13 +1,13 @@ use crate::build; use crate::config::{Channel, ConfigInfo}; use crate::utils::{ - get_toolchain, git_clone, git_clone_root_dir, remove_file, run_command, run_command_with_env, + create_dir, get_toolchain, git_clone, git_clone_root_dir, remove_file, run_command, run_command_with_env, run_command_with_output_and_env, rustc_version_info, split_args, walk_dir, }; use std::collections::{BTreeSet, HashMap}; use std::ffi::OsStr; -use std::fs::{create_dir_all, remove_dir_all, File}; +use std::fs::{remove_dir_all, File}; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -211,8 +211,7 @@ fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { fn clean(_env: &Env, args: &TestArg) -> Result<(), String> { let _ = std::fs::remove_dir_all(&args.config_info.cargo_target_dir); let path = Path::new(&args.config_info.cargo_target_dir).join("gccjit"); - std::fs::create_dir_all(&path) - .map_err(|error| format!("failed to create folder `{}`: {:?}", path.display(), error)) + create_dir(&path) } fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { @@ -715,8 +714,7 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> { }; let projects_path = Path::new("projects"); - create_dir_all(projects_path) - .map_err(|err| format!("Failed to create directory `projects`: {}", err))?; + create_dir(projects_path)?; let nb_parts = args.nb_parts.unwrap_or(0); if nb_parts > 0 { diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index d9c13fd143d15..cd7c035e690c6 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -307,6 +307,12 @@ pub fn git_clone( git_clone_inner(to_clone, dest, shallow_clone, repo_name) } +pub fn create_dir>(path: P) -> Result<(), String> { + fs::create_dir_all(&path).map_err(|error| { + format!("Failed to create directory `{}`: {:?}", path.as_ref().display(), error) + }) +} + /// This function differs from `git_clone` in how it handles *where* the repository will be cloned. /// In `git_clone`, it is cloned in the provided path. In this function, the path you provide is /// the parent folder. So if you pass "a" as folder and try to clone "b.git", it will be cloned into From ab1ea400a8ac464bb6e39b5b3376e2a0bfcdd3df Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 20 Mar 2024 14:18:38 +0100 Subject: [PATCH 3/7] Format code --- build_system/src/build.rs | 25 +---- build_system/src/cargo.rs | 27 +---- build_system/src/clean.rs | 8 +- build_system/src/config.rs | 62 +++-------- build_system/src/prepare.rs | 66 +++-------- build_system/src/test.rs | 214 +++++++++--------------------------- build_system/src/utils.rs | 50 ++------- 7 files changed, 99 insertions(+), 353 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index a4ea5107a0505..5d3e355a52516 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -151,15 +151,7 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu // Copy the source files to the sysroot (Rust for Linux needs this). let sysroot_src_path = start_dir.join("sysroot/lib/rustlib/src/rust"); create_dir(&sysroot_src_path)?; - run_command( - &[ - &"cp", - &"-r", - &start_dir.join("sysroot_src/library/"), - &sysroot_src_path, - ], - None, - )?; + run_command(&[&"cp", &"-r", &start_dir.join("sysroot_src/library/"), &sysroot_src_path], None)?; Ok(()) } @@ -167,20 +159,11 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu fn build_codegen(args: &mut BuildArg) -> Result<(), String> { let mut env = HashMap::new(); - env.insert( - "LD_LIBRARY_PATH".to_string(), - args.config_info.gcc_path.clone(), - ); - env.insert( - "LIBRARY_PATH".to_string(), - args.config_info.gcc_path.clone(), - ); + env.insert("LD_LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone()); + env.insert("LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone()); if args.config_info.no_default_features { - env.insert( - "RUSTFLAGS".to_string(), - "-Csymbol-mangling-version=v0".to_string(), - ); + env.insert("RUSTFLAGS".to_string(), "-Csymbol-mangling-version=v0".to_string()); } let mut command: Vec<&dyn AsRef> = vec![&"cargo", &"rustc"]; diff --git a/build_system/src/cargo.rs b/build_system/src/cargo.rs index 1cfcdba6b1cd1..e4ea79b06f221 100644 --- a/build_system/src/cargo.rs +++ b/build_system/src/cargo.rs @@ -16,9 +16,7 @@ fn args() -> Result>, String> { } let args = std::env::args().skip(2).collect::>(); if args.is_empty() { - return Err( - "Expected at least one argument for `cargo` subcommand, found none".to_string(), - ); + return Err("Expected at least one argument for `cargo` subcommand, found none".to_string()); } Ok(Some(args)) } @@ -48,17 +46,10 @@ pub fn run() -> Result<(), String> { let current_exe = std::env::current_exe() .and_then(|path| path.canonicalize()) .map_err(|error| format!("Failed to get current exe path: {:?}", error))?; - let mut parent_dir = current_exe - .components() - .map(|comp| comp.as_os_str()) - .collect::>(); + let mut parent_dir = current_exe.components().map(|comp| comp.as_os_str()).collect::>(); // We run this script from "build_system/target/release/y", so we need to remove these elements. for to_remove in &["y", "release", "target", "build_system"] { - if parent_dir - .last() - .map(|part| part == to_remove) - .unwrap_or(false) - { + if parent_dir.last().map(|part| part == to_remove).unwrap_or(false) { parent_dir.pop(); } else { return Err(format!( @@ -69,11 +60,7 @@ pub fn run() -> Result<(), String> { } let parent_dir = PathBuf::from(parent_dir.join(&OsStr::new("/"))); std::env::set_current_dir(&parent_dir).map_err(|error| { - format!( - "Failed to go to `{}` folder: {:?}", - parent_dir.display(), - error - ) + format!("Failed to go to `{}` folder: {:?}", parent_dir.display(), error) })?; let mut env: HashMap = std::env::vars().collect(); @@ -92,11 +79,7 @@ pub fn run() -> Result<(), String> { // We go back to the original folder since we now have set up everything we needed. std::env::set_current_dir(¤t_dir).map_err(|error| { - format!( - "Failed to go back to `{}` folder: {:?}", - current_dir.display(), - error - ) + format!("Failed to go back to `{}` folder: {:?}", current_dir.display(), error) })?; let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); diff --git a/build_system/src/clean.rs b/build_system/src/clean.rs index cd8e691a0ed9f..22055ab9841d4 100644 --- a/build_system/src/clean.rs +++ b/build_system/src/clean.rs @@ -42,12 +42,8 @@ fn usage() { } fn clean_all() -> Result<(), String> { - let dirs_to_remove = [ - "target", - "build_sysroot/sysroot", - "build_sysroot/sysroot_src", - "build_sysroot/target", - ]; + let dirs_to_remove = + ["target", "build_sysroot/sysroot", "build_sysroot/sysroot_src", "build_sysroot/target"]; for dir in dirs_to_remove { let _ = remove_dir_all(dir); } diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 2e6d1b038c108..274767ea0b17a 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -1,5 +1,6 @@ use crate::utils::{ - create_dir, create_symlink, get_os_name, run_command_with_output, rustc_version_info, split_args, + create_dir, create_symlink, get_os_name, run_command_with_output, rustc_version_info, + split_args, }; use std::collections::HashMap; use std::env as std_env; @@ -26,11 +27,7 @@ impl Channel { } fn failed_config_parsing(config_file: &Path, err: &str) -> Result { - Err(format!( - "Failed to parse `{}`: {}", - config_file.display(), - err - )) + Err(format!("Failed to parse `{}`: {}", config_file.display(), err)) } #[derive(Default)] @@ -48,11 +45,7 @@ impl ConfigFile { ) })?; let toml = Toml::parse(&content).map_err(|err| { - format!( - "Error occurred around `{}`: {:?}", - &content[err.start..=err.end], - err.kind - ) + format!("Error occurred around `{}`: {:?}", &content[err.start..=err.end], err.kind) })?; let mut config = Self::default(); for (key, value) in toml.iter() { @@ -181,11 +174,7 @@ impl ConfigInfo { }, "--use-backend" => match args.next() { Some(backend) if !backend.is_empty() => self.backend = Some(backend), - _ => { - return Err( - "Expected an argument after `--use-backend`, found nothing".into() - ) - } + _ => return Err("Expected an argument after `--use-backend`, found nothing".into()), }, "--no-default-features" => self.no_default_features = true, _ => return Ok(false), @@ -231,11 +220,7 @@ impl ConfigInfo { create_dir(&output_dir)?; } let output_dir = output_dir.canonicalize().map_err(|err| { - format!( - "Failed to get absolute path of `{}`: {:?}", - output_dir.display(), - err - ) + format!("Failed to get absolute path of `{}`: {:?}", output_dir.display(), err) })?; let libgccjit_so_name = "libgccjit.so"; @@ -269,10 +254,7 @@ impl ConfigInfo { println!("Downloaded libgccjit.so version {} successfully!", commit); // We need to create a link named `libgccjit.so.0` because that's what the linker is // looking for. - create_symlink( - &libgccjit_so, - output_dir.join(&format!("{}.0", libgccjit_so_name)), - )?; + create_symlink(&libgccjit_so, output_dir.join(&format!("{}.0", libgccjit_so_name)))?; } self.gcc_path = output_dir.display().to_string(); @@ -292,10 +274,7 @@ impl ConfigInfo { Some(config_file) => config_file.into(), None => self.compute_path("config.toml"), }; - let ConfigFile { - gcc_path, - download_gccjit, - } = ConfigFile::new(&config_file)?; + let ConfigFile { gcc_path, download_gccjit } = ConfigFile::new(&config_file)?; if let Some(true) = download_gccjit { self.download_gccjit_if_needed()?; @@ -304,10 +283,7 @@ impl ConfigInfo { self.gcc_path = match gcc_path { Some(path) => path, None => { - return Err(format!( - "missing `gcc-path` value from `{}`", - config_file.display(), - )) + return Err(format!("missing `gcc-path` value from `{}`", config_file.display(),)) } }; Ok(()) @@ -387,17 +363,15 @@ impl ConfigInfo { .join(&format!("librustc_codegen_gcc.{}", self.dylib_ext)) .display() .to_string(); - self.sysroot_path = current_dir - .join("build_sysroot/sysroot") - .display() - .to_string(); + self.sysroot_path = current_dir.join("build_sysroot/sysroot").display().to_string(); if let Some(backend) = &self.backend { // This option is only used in the rust compiler testsuite. The sysroot is handled // by its build system directly so no need to set it ourselves. rustflags.push(format!("-Zcodegen-backend={}", backend)); } else { rustflags.extend_from_slice(&[ - "--sysroot".to_string(), self.sysroot_path.clone(), + "--sysroot".to_string(), + self.sysroot_path.clone(), format!("-Zcodegen-backend={}", self.cg_backend_path), ]); } @@ -436,10 +410,8 @@ impl ConfigInfo { // display metadata load errors env.insert("RUSTC_LOG".to_string(), "warn".to_string()); - let sysroot = current_dir.join(&format!( - "build_sysroot/sysroot/lib/rustlib/{}/lib", - self.target_triple, - )); + let sysroot = current_dir + .join(&format!("build_sysroot/sysroot/lib/rustlib/{}/lib", self.target_triple,)); let ld_library_path = format!( "{target}:{sysroot}:{gcc_path}", target = self.cargo_target_dir, @@ -517,11 +489,7 @@ fn download_gccjit( &"--retry", &"3", &"-SRfL", - if with_progress_bar { - &"--progress-bar" - } else { - &"-s" - }, + if with_progress_bar { &"--progress-bar" } else { &"-s" }, &url.as_str(), ], Some(&output_dir), diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs index fd577fa94341f..4b6756a2eefbe 100644 --- a/build_system/src/prepare.rs +++ b/build_system/src/prepare.rs @@ -1,6 +1,7 @@ use crate::rustc_info::get_rustc_path; use crate::utils::{ - cargo_install, create_dir, git_clone_root_dir, remove_file, run_command, run_command_with_output, walk_dir, + cargo_install, create_dir, git_clone_root_dir, remove_file, run_command, + run_command_with_output, walk_dir, }; use std::fs; @@ -32,21 +33,14 @@ fn prepare_libcore( let sysroot_dir = sysroot_path.join("sysroot_src"); if sysroot_dir.is_dir() { if let Err(error) = fs::remove_dir_all(&sysroot_dir) { - return Err(format!( - "Failed to remove `{}`: {:?}", - sysroot_dir.display(), - error, - )); + return Err(format!("Failed to remove `{}`: {:?}", sysroot_dir.display(), error,)); } } let sysroot_library_dir = sysroot_dir.join("library"); create_dir(&sysroot_library_dir)?; - run_command( - &[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir], - None, - )?; + run_command(&[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir], None)?; println!("[GIT] init (cwd): `{}`", sysroot_dir.display()); run_command(&[&"git", &"init"], Some(&sysroot_dir))?; @@ -57,26 +51,11 @@ fn prepare_libcore( // This is needed on systems where nothing is configured. // git really needs something here, or it will fail. // Even using --author is not enough. - run_command( - &[&"git", &"config", &"user.email", &"none@example.com"], - Some(&sysroot_dir), - )?; - run_command( - &[&"git", &"config", &"user.name", &"None"], - Some(&sysroot_dir), - )?; - run_command( - &[&"git", &"config", &"core.autocrlf", &"false"], - Some(&sysroot_dir), - )?; - run_command( - &[&"git", &"config", &"commit.gpgSign", &"false"], - Some(&sysroot_dir), - )?; - run_command( - &[&"git", &"commit", &"-m", &"Initial commit", &"-q"], - Some(&sysroot_dir), - )?; + run_command(&[&"git", &"config", &"user.email", &"none@example.com"], Some(&sysroot_dir))?; + run_command(&[&"git", &"config", &"user.name", &"None"], Some(&sysroot_dir))?; + run_command(&[&"git", &"config", &"core.autocrlf", &"false"], Some(&sysroot_dir))?; + run_command(&[&"git", &"config", &"commit.gpgSign", &"false"], Some(&sysroot_dir))?; + run_command(&[&"git", &"commit", &"-m", &"Initial commit", &"-q"], Some(&sysroot_dir))?; let mut patches = Vec::new(); walk_dir( @@ -114,13 +93,7 @@ fn prepare_libcore( run_command_with_output(&[&"git", &"apply", &path], Some(&sysroot_dir))?; run_command_with_output(&[&"git", &"add", &"-A"], Some(&sysroot_dir))?; run_command_with_output( - &[ - &"git", - &"commit", - &"--no-gpg-sign", - &"-m", - &format!("Patch {}", path.display()), - ], + &[&"git", &"commit", &"--no-gpg-sign", &"-m", &format!("Patch {}", path.display())], Some(&sysroot_dir), )?; } @@ -139,13 +112,7 @@ fn prepare_rand() -> Result<(), String> { run_command_with_output(&[&"git", &"apply", &path], Some(rand_dir))?; run_command_with_output(&[&"git", &"add", &"-A"], Some(rand_dir))?; run_command_with_output( - &[ - &"git", - &"commit", - &"--no-gpg-sign", - &"-m", - &format!("Patch {}", path.display()), - ], + &[&"git", &"commit", &"--no-gpg-sign", &"-m", &format!("Patch {}", path.display())], Some(rand_dir), )?; @@ -159,10 +126,7 @@ fn build_raytracer(repo_dir: &Path) -> Result<(), String> { if mv_target.is_file() { remove_file(&mv_target)?; } - run_command( - &[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], - Some(repo_dir), - )?; + run_command(&[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], Some(repo_dir))?; Ok(()) } @@ -207,11 +171,7 @@ impl PrepareArg { a => return Err(format!("Unknown argument `{a}`")), } } - Ok(Some(Self { - cross_compile, - only_libcore, - libgccjit12_patches, - })) + Ok(Some(Self { cross_compile, only_libcore, libgccjit12_patches })) } fn usage() { diff --git a/build_system/src/test.rs b/build_system/src/test.rs index c0cb6c8f2397b..b27f28795d7b6 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1,8 +1,9 @@ use crate::build; use crate::config::{Channel, ConfigInfo}; use crate::utils::{ - create_dir, get_toolchain, git_clone, git_clone_root_dir, remove_file, run_command, run_command_with_env, - run_command_with_output_and_env, rustc_version_info, split_args, walk_dir, + create_dir, get_toolchain, git_clone, git_clone_root_dir, remove_file, run_command, + run_command_with_env, run_command_with_output_and_env, rustc_version_info, split_args, + walk_dir, }; use std::collections::{BTreeSet, HashMap}; @@ -19,46 +20,23 @@ type Runners = HashMap<&'static str, (&'static str, Runner)>; fn get_runners() -> Runners { let mut runners = HashMap::new(); - runners.insert( - "--test-rustc", - ("Run all rustc tests", test_rustc as Runner), - ); - runners.insert( - "--test-successful-rustc", - ("Run successful rustc tests", test_successful_rustc), - ); - runners.insert( - "--test-failing-rustc", - ("Run failing rustc tests", test_failing_rustc), - ); - runners.insert( - "--projects", - ("Run the tests of popular crates", test_projects), - ); + runners.insert("--test-rustc", ("Run all rustc tests", test_rustc as Runner)); + runners + .insert("--test-successful-rustc", ("Run successful rustc tests", test_successful_rustc)); + runners.insert("--test-failing-rustc", ("Run failing rustc tests", test_failing_rustc)); + runners.insert("--projects", ("Run the tests of popular crates", test_projects)); runners.insert("--test-libcore", ("Run libcore tests", test_libcore)); runners.insert("--clean", ("Empty cargo target directory", clean)); runners.insert("--build-sysroot", ("Build sysroot", build_sysroot)); runners.insert("--std-tests", ("Run std tests", std_tests)); runners.insert("--asm-tests", ("Run asm tests", asm_tests)); - runners.insert( - "--extended-tests", - ("Run extended sysroot tests", extended_sysroot_tests), - ); - runners.insert( - "--extended-rand-tests", - ("Run extended rand tests", extended_rand_tests), - ); + runners.insert("--extended-tests", ("Run extended sysroot tests", extended_sysroot_tests)); + runners.insert("--extended-rand-tests", ("Run extended rand tests", extended_rand_tests)); runners.insert( "--extended-regex-example-tests", - ( - "Run extended regex example tests", - extended_regex_example_tests, - ), - ); - runners.insert( - "--extended-regex-tests", - ("Run extended regex tests", extended_regex_tests), + ("Run extended regex example tests", extended_regex_example_tests), ); + runners.insert("--extended-regex-tests", ("Run extended regex tests", extended_regex_tests)); runners.insert("--mini-tests", ("Run mini tests", mini_tests)); runners @@ -71,15 +49,9 @@ fn get_number_after_arg( match args.next() { Some(nb) if !nb.is_empty() => match usize::from_str(&nb) { Ok(nb) => Ok(nb), - Err(_) => Err(format!( - "Expected a number after `{}`, found `{}`", - option, nb - )), + Err(_) => Err(format!("Expected a number after `{}`, found `{}`", option, nb)), }, - _ => Err(format!( - "Expected a number after `{}`, found nothing", - option - )), + _ => Err(format!("Expected a number after `{}`, found nothing", option)), } } @@ -130,9 +102,7 @@ impl TestArg { match arg.as_str() { "--features" => match args.next() { Some(feature) if !feature.is_empty() => { - test_arg - .flags - .extend_from_slice(&["--features".into(), feature]); + test_arg.flags.extend_from_slice(&["--features".into(), feature]); } _ => { return Err("Expected an argument after `--features`, found nothing".into()) @@ -303,13 +273,8 @@ fn maybe_run_command_in_vm( let sudo_command: &[&dyn AsRef] = &[&"sudo", &"cp", &exe, &vm_exe_path]; run_command_with_env(sudo_command, None, Some(env))?; - let mut vm_command: Vec<&dyn AsRef> = vec![ - &"sudo", - &"chroot", - &vm_dir, - &"qemu-m68k-static", - &inside_vm_exe_path, - ]; + let mut vm_command: Vec<&dyn AsRef> = + vec![&"sudo", &"chroot", &vm_dir, &"qemu-m68k-static", &inside_vm_exe_path]; vm_command.extend_from_slice(command); run_command_with_output_and_env(&vm_command, Some(&vm_parent_dir), Some(env))?; Ok(()) @@ -398,11 +363,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { } run_command_with_env(&command, None, Some(env))?; maybe_run_command_in_vm( - &[ - &cargo_target_dir.join("std_example"), - &"--target", - &args.config_info.target_triple, - ], + &[&cargo_target_dir.join("std_example"), &"--target", &args.config_info.target_triple], env, args, )?; @@ -426,11 +387,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { command.push(test_flag); } run_command_with_env(&command, None, Some(env))?; - maybe_run_command_in_vm( - &[&cargo_target_dir.join("subslice-patterns-const-eval")], - env, - args, - )?; + maybe_run_command_in_vm(&[&cargo_target_dir.join("subslice-patterns-const-eval")], env, args)?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] track-caller-attribute"); @@ -446,11 +403,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { command.push(test_flag); } run_command_with_env(&command, None, Some(env))?; - maybe_run_command_in_vm( - &[&cargo_target_dir.join("track-caller-attribute")], - env, - args, - )?; + maybe_run_command_in_vm(&[&cargo_target_dir.join("track-caller-attribute")], env, args)?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] mod_bench"); @@ -476,11 +429,7 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result { ); let rust_dir_path = Path::new(crate::BUILD_DIR).join("rust"); // If the repository was already cloned, command will fail, so doesn't matter. - let _ = git_clone( - "https://github.com/rust-lang/rust.git", - Some(&rust_dir_path), - false, - ); + let _ = git_clone("https://github.com/rust-lang/rust.git", Some(&rust_dir_path), false); let rust_dir: Option<&Path> = Some(&rust_dir_path); run_command(&[&"git", &"checkout", &"--", &"tests/"], rust_dir)?; run_command_with_output_and_env(&[&"git", &"fetch"], rust_dir, Some(env))?; @@ -510,12 +459,8 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result { } })?; let rustc = String::from_utf8( - run_command_with_env( - &[&"rustup", &toolchain, &"which", &"rustc"], - rust_dir, - Some(env), - )? - .stdout, + run_command_with_env(&[&"rustup", &toolchain, &"which", &"rustc"], rust_dir, Some(env))? + .stdout, ) .map_err(|error| format!("Failed to retrieve rustc path: {:?}", error)) .and_then(|rustc| { @@ -572,13 +517,7 @@ download-ci-llvm = false llvm_filecheck = llvm_filecheck.trim(), ), ) - .map_err(|error| { - format!( - "Failed to write into `{}`: {:?}", - file_path.display(), - error - ) - })?; + .map_err(|error| format!("Failed to write into `{}`: {:?}", file_path.display(), error))?; Ok(rust_dir_path) } @@ -590,11 +529,8 @@ fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); - let extra = if args.is_using_gcc_master_branch() { - "" - } else { - " -Csymbol-mangling-version=v0" - }; + let extra = + if args.is_using_gcc_master_branch() { "" } else { " -Csymbol-mangling-version=v0" }; let rustc_args = &format!( r#"-Zpanic-abort-tests \ @@ -761,10 +697,8 @@ fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { } let mut env = env.clone(); // newer aho_corasick versions throw a deprecation warning - let rustflags = format!( - "{} --cap-lints warn", - env.get("RUSTFLAGS").cloned().unwrap_or_default() - ); + let rustflags = + format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default()); env.insert("RUSTFLAGS".to_string(), rustflags); let path = Path::new(crate::BUILD_DIR).join("rand"); @@ -786,18 +720,11 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> println!("[TEST] rust-lang/regex example shootout-regex-dna"); let mut env = env.clone(); // newer aho_corasick versions throw a deprecation warning - let rustflags = format!( - "{} --cap-lints warn", - env.get("RUSTFLAGS").cloned().unwrap_or_default() - ); + let rustflags = + format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default()); env.insert("RUSTFLAGS".to_string(), rustflags); // Make sure `[codegen mono items] start` doesn't poison the diff - run_cargo_command( - &[&"build", &"--example", &"shootout-regex-dna"], - Some(&path), - &env, - args, - )?; + run_cargo_command(&[&"build", &"--example", &"shootout-regex-dna"], Some(&path), &env, args)?; run_cargo_command_with_callback( &[&"run", &"--example", &"shootout-regex-dna"], @@ -808,10 +735,8 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> // FIXME: rewrite this with `child.stdin.write_all()` because // `examples/regexdna-input.txt` is very small. let mut command: Vec<&dyn AsRef> = vec![&"bash", &"-c"]; - let cargo_args = cargo_command - .iter() - .map(|s| s.as_ref().to_str().unwrap()) - .collect::>(); + let cargo_args = + cargo_command.iter().map(|s| s.as_ref().to_str().unwrap()).collect::>(); let bash_command = format!( "cat examples/regexdna-input.txt | {} | grep -v 'Spawned thread' > res.txt", cargo_args.join(" "), @@ -839,10 +764,8 @@ fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { println!("[TEST] rust-lang/regex tests"); let mut env = env.clone(); // newer aho_corasick versions throw a deprecation warning - let rustflags = format!( - "{} --cap-lints warn", - env.get("RUSTFLAGS").cloned().unwrap_or_default() - ); + let rustflags = + format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default()); env.insert("RUSTFLAGS".to_string(), rustflags); let path = Path::new(crate::BUILD_DIR).join("regex"); run_cargo_command( @@ -897,6 +820,7 @@ fn should_not_remove_test(file: &str) -> bool { .any(|to_ignore| file.ends_with(to_ignore)) } +#[rustfmt::skip] fn should_remove_test(file_path: &Path) -> Result { // Tests generating errors. let file = File::open(file_path) @@ -914,8 +838,8 @@ fn should_remove_test(file_path: &Path) -> Result { "//~", "thread", ] - .iter() - .any(|check| line.contains(check)) + .iter() + .any(|check| line.contains(check)) { return Ok(true); } @@ -923,11 +847,7 @@ fn should_remove_test(file_path: &Path) -> Result { return Ok(true); } } - if file_path - .display() - .to_string() - .contains("ambiguous-4-extern.rs") - { + if file_path.display().to_string().contains("ambiguous-4-extern.rs") { eprintln!("nothing found for {file_path:?}"); } Ok(false) @@ -970,21 +890,13 @@ where // These two functions are used to remove files that are known to not be working currently // with the GCC backend to reduce noise. fn dir_handling(dir: &Path) -> Result<(), String> { - if dir - .file_name() - .map(|name| name == "auxiliary") - .unwrap_or(true) - { + if dir.file_name().map(|name| name == "auxiliary").unwrap_or(true) { return Ok(()); } walk_dir(dir, dir_handling, file_handling) } fn file_handling(file_path: &Path) -> Result<(), String> { - if !file_path - .extension() - .map(|extension| extension == "rs") - .unwrap_or(false) - { + if !file_path.extension().map(|extension| extension == "rs").unwrap_or(false) { return Ok(()); } let path_str = file_path.display().to_string().replace("\\", "/"); @@ -1017,10 +929,7 @@ where if nb_parts > 0 { let current_part = args.current_part.unwrap(); // FIXME: create a function "display_if_not_quiet" or something along the line. - println!( - "Splitting ui_test into {} parts (and running part {})", - nb_parts, current_part - ); + println!("Splitting ui_test into {} parts (and running part {})", nb_parts, current_part); let out = String::from_utf8( run_command( &[ @@ -1060,11 +969,8 @@ where println!("[TEST] rustc test suite"); env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); - let extra = if args.is_using_gcc_master_branch() { - "" - } else { - " -Csymbol-mangling-version=v0" - }; + let extra = + if args.is_using_gcc_master_branch() { "" } else { " -Csymbol-mangling-version=v0" }; let rustc_args = format!( "{} -Zcodegen-backend={} --sysroot {}{}", @@ -1118,18 +1024,11 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { // Putting back only the failing ones. let path = "tests/failing-ui-tests.txt"; if let Ok(files) = std::fs::read_to_string(path) { - for file in files - .split('\n') - .map(|line| line.trim()) - .filter(|line| !line.is_empty()) - { + for file in files.split('\n').map(|line| line.trim()).filter(|line| !line.is_empty()) { run_command(&[&"git", &"checkout", &"--", &file], Some(&rust_path))?; } } else { - println!( - "Failed to read `{}`, not putting back failing ui tests", - path - ); + println!("Failed to read `{}`, not putting back failing ui tests", path); } Ok(true) }) @@ -1140,19 +1039,12 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { // Removing the failing tests. let path = "tests/failing-ui-tests.txt"; if let Ok(files) = std::fs::read_to_string(path) { - for file in files - .split('\n') - .map(|line| line.trim()) - .filter(|line| !line.is_empty()) - { + for file in files.split('\n').map(|line| line.trim()).filter(|line| !line.is_empty()) { let path = rust_path.join(file); remove_file(&path)?; } } else { - println!( - "Failed to read `{}`, not putting back failing ui tests", - path - ); + println!("Failed to read `{}`, not putting back failing ui tests", path); } Ok(true) }) @@ -1179,14 +1071,8 @@ pub fn run() -> Result<(), String> { if !args.use_system_gcc { args.config_info.setup_gcc_path()?; - env.insert( - "LIBRARY_PATH".to_string(), - args.config_info.gcc_path.clone(), - ); - env.insert( - "LD_LIBRARY_PATH".to_string(), - args.config_info.gcc_path.clone(), - ); + env.insert("LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone()); + env.insert("LD_LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone()); } build_if_no_backend(&env, &args)?; diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index cd7c035e690c6..c5e7f532a7ddc 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -37,13 +37,8 @@ fn check_exit_status( } let mut error = format!( "Command `{}`{} exited with status {:?}", - input - .iter() - .map(|s| s.as_ref().to_str().unwrap()) - .collect::>() - .join(" "), - cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())) - .unwrap_or_default(), + input.iter().map(|s| s.as_ref().to_str().unwrap()).collect::>().join(" "), + cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())).unwrap_or_default(), exit_status.code() ); let input = input.iter().map(|i| i.as_ref()).collect::>(); @@ -68,11 +63,7 @@ fn check_exit_status( fn command_error(input: &[&dyn AsRef], cwd: &Option<&Path>, error: D) -> String { format!( "Command `{}`{} failed to run: {error:?}", - input - .iter() - .map(|s| s.as_ref().to_str().unwrap()) - .collect::>() - .join(" "), + input.iter().map(|s| s.as_ref().to_str().unwrap()).collect::>().join(" "), cwd.as_ref() .map(|cwd| format!(" (running in folder `{}`)", cwd.display(),)) .unwrap_or_default(), @@ -88,9 +79,8 @@ pub fn run_command_with_env( cwd: Option<&Path>, env: Option<&HashMap>, ) -> Result { - let output = get_command_inner(input, cwd, env) - .output() - .map_err(|e| command_error(input, &cwd, e))?; + let output = + get_command_inner(input, cwd, env).output().map_err(|e| command_error(input, &cwd, e))?; check_exit_status(input, cwd, output.status, Some(&output), true)?; Ok(output) } @@ -164,10 +154,7 @@ pub fn cargo_install(to_install: &str) -> Result<(), String> { pub fn get_os_name() -> Result { let output = run_command(&[&"uname"], None)?; - let name = std::str::from_utf8(&output.stdout) - .unwrap_or("") - .trim() - .to_string(); + let name = std::str::from_utf8(&output.stdout).unwrap_or("").trim().to_string(); if !name.is_empty() { Ok(name) } else { @@ -274,11 +261,7 @@ fn git_clone_inner( command.push(&"1"); } run_command_with_output(&command, None)?; - Ok(CloneResult { - ran_clone: true, - repo_name, - repo_dir: dest.display().to_string(), - }) + Ok(CloneResult { ran_clone: true, repo_name, repo_dir: dest.display().to_string() }) } fn get_repo_name(url: &str) -> String { @@ -324,12 +307,7 @@ pub fn git_clone_root_dir( ) -> Result { let repo_name = get_repo_name(to_clone); - git_clone_inner( - to_clone, - &dest_parent_dir.join(&repo_name), - shallow_clone, - repo_name, - ) + git_clone_inner(to_clone, &dest_parent_dir.join(&repo_name), shallow_clone, repo_name) } pub fn walk_dir(dir: P, mut dir_cb: D, mut file_cb: F) -> Result<(), String> @@ -389,11 +367,7 @@ pub fn split_args(args: &str) -> Result, String> { } } if !found_end { - return Err(format!( - "Didn't find `{}` at the end of `{}`", - end, - &args[start..] - )); + return Err(format!("Didn't find `{}` at the end of `{}`", end, &args[start..])); } } else if c == '\\' { // We skip the escaped character. @@ -409,11 +383,7 @@ pub fn split_args(args: &str) -> Result, String> { pub fn remove_file + ?Sized>(file_path: &P) -> Result<(), String> { std::fs::remove_file(file_path).map_err(|error| { - format!( - "Failed to remove `{}`: {:?}", - file_path.as_ref().display(), - error - ) + format!("Failed to remove `{}`: {:?}", file_path.as_ref().display(), error) }) } From cde105a651a3f3249d16780f4ff0173ec8f6b87b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 20 Mar 2024 15:15:20 +0100 Subject: [PATCH 4/7] Move `build_sysroot` folder into `build_system` --- .github/workflows/release.yml | 4 ++-- .github/workflows/stdarch.yml | 8 +++---- .gitignore | 4 ---- .../build_sysroot}/Cargo.toml | 2 +- .../src => build_system/build_sysroot}/lib.rs | 0 build_system/src/build.rs | 24 +++++++++++++++---- build_system/src/clean.rs | 16 +++++++++---- build_system/src/config.rs | 12 +++++----- build_system/src/prepare.rs | 8 +++---- build_system/src/test.rs | 13 +++++----- build_system/src/utils.rs | 17 +++++++++++++ tests/lang_tests_common.rs | 2 +- 12 files changed, 73 insertions(+), 37 deletions(-) rename {build_sysroot => build_system/build_sysroot}/Cargo.toml (93%) rename {build_sysroot/src => build_system/build_sysroot}/lib.rs (100%) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28336998ffcd7..60a71d7d3dd78 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,12 +62,12 @@ jobs: git config --global user.email "user@example.com" git config --global user.name "User" ./y.sh prepare - # FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros. - echo -n 'lto = "fat"' >> build_sysroot/Cargo.toml - name: Add more failing tests because of undefined symbol errors (FIXME) run: cat tests/failing-lto-tests.txt >> tests/failing-ui-tests.txt - name: Run tests run: | + # FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros. + echo -n 'lto = "fat"' >> build_system/build_sysroot/Cargo.toml EMBED_LTO_BITCODE=1 ./y.sh test --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }} diff --git a/.github/workflows/stdarch.yml b/.github/workflows/stdarch.yml index 41a9318007f15..25eb5c1b69f28 100644 --- a/.github/workflows/stdarch.yml +++ b/.github/workflows/stdarch.yml @@ -89,12 +89,12 @@ jobs: - name: Run stdarch tests if: ${{ !matrix.cargo_runner }} run: | - cd build_sysroot/sysroot_src/library/stdarch/ - CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../y.sh cargo test + cd build/build_sysroot/sysroot_src/library/stdarch/ + CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../../y.sh cargo test - name: Run stdarch tests if: ${{ matrix.cargo_runner }} run: | - cd build_sysroot/sysroot_src/library/stdarch/ + cd build/build_sysroot/sysroot_src/library/stdarch/ # FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro. - STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../y.sh cargo test -- --skip rtm --skip tbm --skip sse4a + STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../../y.sh cargo test -- --skip rtm --skip tbm --skip sse4a diff --git a/.gitignore b/.gitignore index bf975f92014da..c1e6631a281ba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,6 @@ perf.data perf.data.old *.events *.string* -/build_sysroot/sysroot -/build_sysroot/sysroot_src -/build_sysroot/Cargo.lock -/build_sysroot/test_target/Cargo.lock gimple* *asm res diff --git a/build_sysroot/Cargo.toml b/build_system/build_sysroot/Cargo.toml similarity index 93% rename from build_sysroot/Cargo.toml rename to build_system/build_sysroot/Cargo.toml index f46c9c8ce157e..05503128f2af0 100644 --- a/build_sysroot/Cargo.toml +++ b/build_system/build_sysroot/Cargo.toml @@ -1,5 +1,5 @@ [package] -authors = ["bjorn3 "] +authors = ["rustc_codegen_gcc devs"] name = "sysroot" version = "0.0.0" resolver = "2" diff --git a/build_sysroot/src/lib.rs b/build_system/build_sysroot/lib.rs similarity index 100% rename from build_sysroot/src/lib.rs rename to build_system/build_sysroot/lib.rs diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 5d3e355a52516..6313e4ffccb97 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -1,5 +1,7 @@ use crate::config::{Channel, ConfigInfo}; -use crate::utils::{create_dir, run_command, run_command_with_output_and_env, walk_dir}; +use crate::utils::{ + copy_file, create_dir, get_sysroot_dir, run_command, run_command_with_output_and_env, walk_dir, +}; use std::collections::HashMap; use std::ffi::OsStr; use std::fs; @@ -101,10 +103,24 @@ fn cleanup_sysroot_previous_build(start_dir: &Path) { let _ = fs::remove_dir_all(start_dir.join("sysroot")); } +pub fn create_build_sysroot_content(start_dir: &Path) -> Result<(), String> { + if !start_dir.is_dir() { + create_dir(start_dir)?; + } + copy_file("build_system/build_sysroot/Cargo.toml", &start_dir.join("Cargo.toml"))?; + + let src_dir = start_dir.join("src"); + if !src_dir.is_dir() { + create_dir(&src_dir)?; + } + copy_file("build_system/build_sysroot/lib.rs", &start_dir.join("src/lib.rs")) +} + pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Result<(), String> { - let start_dir = Path::new("build_sysroot"); + let start_dir = get_sysroot_dir(); cleanup_sysroot_previous_build(&start_dir); + create_build_sysroot_content(&start_dir)?; // Builds libs let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); @@ -115,7 +131,6 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu if config.no_default_features { rustflags.push_str(" -Csymbol-mangling-version=v0"); } - let mut env = env.clone(); let mut args: Vec<&dyn AsRef> = vec![&"cargo", &"build", &"--target", &config.target]; @@ -132,8 +147,9 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu "debug" }; + let mut env = env.clone(); env.insert("RUSTFLAGS".to_string(), rustflags); - run_command_with_output_and_env(&args, Some(start_dir), Some(&env))?; + run_command_with_output_and_env(&args, Some(&start_dir), Some(&env))?; // Copy files to sysroot let sysroot_path = start_dir.join(format!("sysroot/lib/rustlib/{}/lib/", config.target_triple)); diff --git a/build_system/src/clean.rs b/build_system/src/clean.rs index 22055ab9841d4..55f55acf73ea0 100644 --- a/build_system/src/clean.rs +++ b/build_system/src/clean.rs @@ -1,4 +1,4 @@ -use crate::utils::{remove_file, run_command}; +use crate::utils::{get_sysroot_dir, remove_file, run_command}; use std::fs::remove_dir_all; use std::path::Path; @@ -42,8 +42,13 @@ fn usage() { } fn clean_all() -> Result<(), String> { - let dirs_to_remove = - ["target", "build_sysroot/sysroot", "build_sysroot/sysroot_src", "build_sysroot/target"]; + let build_sysroot = get_sysroot_dir(); + let dirs_to_remove = [ + "target".into(), + build_sysroot.join("sysroot"), + build_sysroot.join("sysroot_src"), + build_sysroot.join("target"), + ]; for dir in dirs_to_remove { let _ = remove_dir_all(dir); } @@ -52,10 +57,11 @@ fn clean_all() -> Result<(), String> { let _ = remove_dir_all(Path::new(crate::BUILD_DIR).join(dir)); } - let files_to_remove = ["build_sysroot/Cargo.lock", "perf.data", "perf.data.old"]; + let files_to_remove = + [build_sysroot.join("Cargo.lock"), "perf.data".into(), "perf.data.old".into()]; for file in files_to_remove { - let _ = remove_file(file); + let _ = remove_file(&file); } println!("Successfully ran `clean all`"); diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 274767ea0b17a..041d75915fb0b 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -1,6 +1,6 @@ use crate::utils::{ - create_dir, create_symlink, get_os_name, run_command_with_output, rustc_version_info, - split_args, + create_dir, create_symlink, get_os_name, get_sysroot_dir, run_command_with_output, + rustc_version_info, split_args, }; use std::collections::HashMap; use std::env as std_env; @@ -363,7 +363,8 @@ impl ConfigInfo { .join(&format!("librustc_codegen_gcc.{}", self.dylib_ext)) .display() .to_string(); - self.sysroot_path = current_dir.join("build_sysroot/sysroot").display().to_string(); + self.sysroot_path = + current_dir.join(&get_sysroot_dir()).join("sysroot").display().to_string(); if let Some(backend) = &self.backend { // This option is only used in the rust compiler testsuite. The sysroot is handled // by its build system directly so no need to set it ourselves. @@ -392,8 +393,6 @@ impl ConfigInfo { rustflags.push("-Csymbol-mangling-version=v0".to_string()); } - - // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. // TODO(antoyo): remove when we can handle ThinLTO. if !env.contains_key(&"FAT_LTO".to_string()) { @@ -411,7 +410,8 @@ impl ConfigInfo { env.insert("RUSTC_LOG".to_string(), "warn".to_string()); let sysroot = current_dir - .join(&format!("build_sysroot/sysroot/lib/rustlib/{}/lib", self.target_triple,)); + .join(&get_sysroot_dir()) + .join(&format!("sysroot/lib/rustlib/{}/lib", self.target_triple)); let ld_library_path = format!( "{target}:{sysroot}:{gcc_path}", target = self.cargo_target_dir, diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs index 4b6756a2eefbe..9f405daa68762 100644 --- a/build_system/src/prepare.rs +++ b/build_system/src/prepare.rs @@ -1,6 +1,6 @@ use crate::rustc_info::get_rustc_path; use crate::utils::{ - cargo_install, create_dir, git_clone_root_dir, remove_file, run_command, + cargo_install, create_dir, get_sysroot_dir, git_clone_root_dir, remove_file, run_command, run_command_with_output, walk_dir, }; @@ -89,7 +89,7 @@ fn prepare_libcore( patches.sort(); for file_path in patches { println!("[GIT] apply `{}`", file_path.display()); - let path = Path::new("../..").join(file_path); + let path = Path::new("../../..").join(file_path); run_command_with_output(&[&"git", &"apply", &path], Some(&sysroot_dir))?; run_command_with_output(&[&"git", &"add", &"-A"], Some(&sysroot_dir))?; run_command_with_output( @@ -192,8 +192,8 @@ pub fn run() -> Result<(), String> { Some(a) => a, None => return Ok(()), }; - let sysroot_path = Path::new("build_sysroot"); - prepare_libcore(sysroot_path, args.libgccjit12_patches, args.cross_compile)?; + let sysroot_path = get_sysroot_dir(); + prepare_libcore(&sysroot_path, args.libgccjit12_patches, args.cross_compile)?; if !args.only_libcore { cargo_install("hyperfine")?; diff --git a/build_system/src/test.rs b/build_system/src/test.rs index b27f28795d7b6..c9cfa13e8759d 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1,9 +1,9 @@ use crate::build; use crate::config::{Channel, ConfigInfo}; use crate::utils::{ - create_dir, get_toolchain, git_clone, git_clone_root_dir, remove_file, run_command, - run_command_with_env, run_command_with_output_and_env, rustc_version_info, split_args, - walk_dir, + create_dir, get_sysroot_dir, get_toolchain, git_clone, git_clone_root_dir, remove_file, + run_command, run_command_with_env, run_command_with_output_and_env, rustc_version_info, + split_args, walk_dir, }; use std::collections::{BTreeSet, HashMap}; @@ -535,12 +535,13 @@ fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { let rustc_args = &format!( r#"-Zpanic-abort-tests \ -Zcodegen-backend="{pwd}/target/{channel}/librustc_codegen_gcc.{dylib_ext}" \ - --sysroot "{pwd}/build_sysroot/sysroot" -Cpanic=abort{extra}"#, + --sysroot "{pwd}/{sysroot_dir}" -Cpanic=abort{extra}"#, pwd = std::env::current_dir() .map_err(|error| format!("`current_dir` failed: {:?}", error))? .display(), channel = args.config_info.channel.as_str(), dylib_ext = args.config_info.dylib_ext, + sysroot_dir = args.config_info.sysroot_path, extra = extra, ); @@ -671,9 +672,9 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> { fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] libcore"); - let path = Path::new("build_sysroot/sysroot_src/library/core/tests"); + let path = get_sysroot_dir().join("sysroot_src/library/core/tests"); let _ = remove_dir_all(path.join("target")); - run_cargo_command(&[&"test"], Some(path), env, args)?; + run_cargo_command(&[&"test"], Some(&path), env, args)?; Ok(()) } diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index c5e7f532a7ddc..45a3ac81b44d4 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -296,6 +296,19 @@ pub fn create_dir>(path: P) -> Result<(), String> { }) } +pub fn copy_file, T: AsRef>(from: F, to: T) -> Result<(), String> { + fs::copy(&from, &to) + .map_err(|error| { + format!( + "Failed to copy file `{}` into `{}`: {:?}", + from.as_ref().display(), + to.as_ref().display(), + error + ) + }) + .map(|_| ()) +} + /// This function differs from `git_clone` in how it handles *where* the repository will be cloned. /// In `git_clone`, it is cloned in the provided path. In this function, the path you provide is /// the parent folder. So if you pass "a" as folder and try to clone "b.git", it will be cloned into @@ -403,6 +416,10 @@ pub fn create_symlink, Q: AsRef>(original: P, link: Q) -> R }) } +pub fn get_sysroot_dir() -> PathBuf { + Path::new(crate::BUILD_DIR).join("build_sysroot") +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/lang_tests_common.rs b/tests/lang_tests_common.rs index d321ffc8ff544..c0fb5fa40733d 100644 --- a/tests/lang_tests_common.rs +++ b/tests/lang_tests_common.rs @@ -79,7 +79,7 @@ pub fn main_inner(profile: Profile) { compiler.args([ &format!("-Zcodegen-backend={}/target/debug/librustc_codegen_gcc.so", current_dir), "--sysroot", - &format!("{}/build_sysroot/sysroot/", current_dir), + &format!("{}/build/build_sysroot/sysroot/", current_dir), "-Zno-parallel-llvm", "-C", "link-arg=-lc", From 52f6d5d5893b74755e044072ebfbaded230c3526 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 21 Mar 2024 17:09:37 +0100 Subject: [PATCH 5/7] Run test commands in the provided order --- build_system/src/test.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index c9cfa13e8759d..86e6f1a95e9f2 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -6,7 +6,7 @@ use crate::utils::{ split_args, walk_dir, }; -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; use std::ffi::OsStr; use std::fs::{remove_dir_all, File}; use std::io::{BufRead, BufReader}; @@ -82,7 +82,7 @@ fn show_usage() { struct TestArg { build_only: bool, use_system_gcc: bool, - runners: BTreeSet, + runners: Vec, flags: Vec, nb_parts: Option, current_part: Option, @@ -127,8 +127,8 @@ impl TestArg { show_usage(); return Ok(None); } - x if runners.contains_key(x) => { - test_arg.runners.insert(x.into()); + x if runners.contains_key(x) && !test_arg.runners.iter().any(|runner| runner == x) => { + test_arg.runners.push(x.into()); } arg => { if !test_arg.config_info.parse_argument(arg, &mut args)? { @@ -535,7 +535,7 @@ fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { let rustc_args = &format!( r#"-Zpanic-abort-tests \ -Zcodegen-backend="{pwd}/target/{channel}/librustc_codegen_gcc.{dylib_ext}" \ - --sysroot "{pwd}/{sysroot_dir}" -Cpanic=abort{extra}"#, + --sysroot "{sysroot_dir}" -Cpanic=abort{extra}"#, pwd = std::env::current_dir() .map_err(|error| format!("`current_dir` failed: {:?}", error))? .display(), @@ -974,11 +974,11 @@ where if args.is_using_gcc_master_branch() { "" } else { " -Csymbol-mangling-version=v0" }; let rustc_args = format!( - "{} -Zcodegen-backend={} --sysroot {}{}", - env.get("TEST_FLAGS").unwrap_or(&String::new()), - args.config_info.cg_backend_path, - args.config_info.sysroot_path, - extra, + "{test_flags} -Zcodegen-backend={backend} --sysroot {sysroot}{extra}", + test_flags = env.get("TEST_FLAGS").unwrap_or(&String::new()), + backend = args.config_info.cg_backend_path, + sysroot = args.config_info.sysroot_path, + extra = extra, ); env.get_mut("RUSTFLAGS").unwrap().clear(); From 7ccd8ce6938cc6f44b3451d7a865b6e0d406d2d4 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 21 Mar 2024 17:46:23 +0100 Subject: [PATCH 6/7] Add fmt check on `build_system` --- .github/workflows/ci.yml | 5 ++++- build_system/src/test.rs | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44730bab4ed15..75694ebd9cea8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,10 @@ jobs: ./y.sh test --release --clean --build-sysroot ${{ matrix.commands }} - name: Check formatting - run: cargo fmt -- --check + run: | + cargo fmt -- --check + cd build_system + cargo fmt -- --check - name: clippy run: | diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 86e6f1a95e9f2..8ce615ff17a45 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -127,7 +127,9 @@ impl TestArg { show_usage(); return Ok(None); } - x if runners.contains_key(x) && !test_arg.runners.iter().any(|runner| runner == x) => { + x if runners.contains_key(x) + && !test_arg.runners.iter().any(|runner| runner == x) => + { test_arg.runners.push(x.into()); } arg => { From da070d356932612ab9564d99d10de7924240566c Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 22 Mar 2024 20:24:44 +0100 Subject: [PATCH 7/7] Clean up `y.sh` path in CI --- .github/workflows/stdarch.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/stdarch.yml b/.github/workflows/stdarch.yml index 25eb5c1b69f28..dc52c94376c5f 100644 --- a/.github/workflows/stdarch.yml +++ b/.github/workflows/stdarch.yml @@ -89,12 +89,10 @@ jobs: - name: Run stdarch tests if: ${{ !matrix.cargo_runner }} run: | - cd build/build_sysroot/sysroot_src/library/stdarch/ - CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../../y.sh cargo test + CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml - name: Run stdarch tests if: ${{ matrix.cargo_runner }} run: | - cd build/build_sysroot/sysroot_src/library/stdarch/ # FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro. - STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../../y.sh cargo test -- --skip rtm --skip tbm --skip sse4a + STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a