Auto merge of rust-lang#129687 - Urgau:rfc3127-sysroot-2, r=jieyouxu

Implement RFC3137 trim-paths sysroot changes - take 2

This PR is a continuation of rust-lang#118149. Nothing really changed, except for rust-lang#129408 which I was able to trigger locally.

Original description:

> Implement parts of rust-lang#111540
>
> Right now, backtraces into sysroot always shows /rustc/$hash in diagnostics, e.g.
>
> ```
> thread 'main' panicked at 'hello world', map-panic.rs:2:50
> stack backtrace:
>    0: std::panicking::begin_panic
>              at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/std/src/panicking.rs:616:12
>    1: map_panic::main::{{closure}}
>              at ./map-panic.rs:2:50
>    2: core::option::Option<T>::map
>              at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/core/src/option.rs:929:29
>    3: map_panic::main
>              at ./map-panic.rs:2:30
>    4: core::ops::function::FnOnce::call_once
>              at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/core/src/ops/function.rs:248:5
> note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
> ```
>
> [RFC 3127 said](https://rust-lang.github.io/rfcs/3127-trim-paths.html#changing-handling-of-sysroot-path-in-rustc)
>
> > We want to change this behaviour such that, when rust-src source files can be discovered, the virtual path is discarded and therefore the local path will be embedded, unless there is a --remap-path-prefix that causes this local path to be remapped in the usual way.
>
> This PR implements this behaviour. When `rust-src` is present at compile time, rustc replaces /rustc/$hash with a real path into local rust-src with best effort. To sanitise this, users must explicitly supply `--remap-path-prefix=<path to rust-src>=foo`.

cc `@cbeuw`
Fix rust-lang#105907
Fix rust-lang#85463

try-job: dist-x86_64-linux
try-job: x86_64-msvc
try-job: dist-x86_64-msvc
try-job: armhf-gnu

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -1622,56 +1622,63 @@ impl<'a> CrateMetadataRef<'a> {
             );
 
             for virtual_dir in virtual_rust_source_base_dir.iter().flatten() {
-                if let Some(real_dir) = &sess.opts.real_rust_source_base_dir {
-                    if let rustc_span::FileName::Real(old_name) = name {
-                        if let rustc_span::RealFileName::Remapped { local_path: _, virtual_name } =
-                            old_name
-                        {
-                            if let Ok(rest) = virtual_name.strip_prefix(virtual_dir) {
-                                let virtual_name = virtual_name.clone();
-
-                                // The std library crates are in
-                                // `$sysroot/lib/rustlib/src/rust/library`, whereas other crates
-                                // may be in `$sysroot/lib/rustlib/src/rust/` directly. So we
-                                // detect crates from the std libs and handle them specially.
-                                const STD_LIBS: &[&str] = &[
-                                    "core",
-                                    "alloc",
-                                    "std",
-                                    "test",
-                                    "term",
-                                    "unwind",
-                                    "proc_macro",
-                                    "panic_abort",
-                                    "panic_unwind",
-                                    "profiler_builtins",
-                                    "rtstartup",
-                                    "rustc-std-workspace-core",
-                                    "rustc-std-workspace-alloc",
-                                    "rustc-std-workspace-std",
-                                    "backtrace",
-                                ];
-                                let is_std_lib = STD_LIBS.iter().any(|l| rest.starts_with(l));
-
-                                let new_path = if is_std_lib {
-                                    real_dir.join("library").join(rest)
-                                } else {
-                                    real_dir.join(rest)
-                                };
-
-                                debug!(
-                                    "try_to_translate_virtual_to_real: `{}` -> `{}`",
-                                    virtual_name.display(),
-                                    new_path.display(),
-                                );
-                                let new_name = rustc_span::RealFileName::Remapped {
-                                    local_path: Some(new_path),
-                                    virtual_name,
-                                };
-                                *old_name = new_name;
-                            }
+                if let Some(real_dir) = &sess.opts.real_rust_source_base_dir
+                    && let rustc_span::FileName::Real(old_name) = name
+                    && let rustc_span::RealFileName::Remapped { local_path: _, virtual_name } =
+                        old_name
+                    && let Ok(rest) = virtual_name.strip_prefix(virtual_dir)
+                {
+                    // The std library crates are in
+                    // `$sysroot/lib/rustlib/src/rust/library`, whereas other crates
+                    // may be in `$sysroot/lib/rustlib/src/rust/` directly. So we
+                    // detect crates from the std libs and handle them specially.
+                    const STD_LIBS: &[&str] = &[
+                        "core",
+                        "alloc",
+                        "std",
+                        "test",
+                        "term",
+                        "unwind",
+                        "proc_macro",
+                        "panic_abort",
+                        "panic_unwind",
+                        "profiler_builtins",
+                        "rtstartup",
+                        "rustc-std-workspace-core",
+                        "rustc-std-workspace-alloc",
+                        "rustc-std-workspace-std",
+                        "backtrace",
+                    ];
+                    let is_std_lib = STD_LIBS.iter().any(|l| rest.starts_with(l));
+
+                    let new_path = if is_std_lib {
+                        real_dir.join("library").join(rest)
+                    } else {
+                        real_dir.join(rest)
+                    };
+
+                    debug!(
+                        "try_to_translate_virtual_to_real: `{}` -> `{}`",
+                        virtual_name.display(),
+                        new_path.display(),
+                    );
+
+                    // Check if the translated real path is affected by any user-requested
+                    // remaps via --remap-path-prefix. Apply them if so.
+                    // Note that this is a special case for imported rust-src paths specified by
+                    // https://rust-lang.github.io/rfcs/3127-trim-paths.html#handling-sysroot-paths.
+                    // Other imported paths are not currently remapped (see #66251).
+                    let (user_remapped, applied) =
+                        sess.source_map().path_mapping().map_prefix(&new_path);
+                    let new_name = if applied {
+                        rustc_span::RealFileName::Remapped {
+                            local_path: Some(new_path.clone()),
+                            virtual_name: user_remapped.to_path_buf(),
                         }
-                    }
+                    } else {
+                        rustc_span::RealFileName::LocalPath(new_path)
+                    };
+                    *old_name = new_name;
                 }
             }
         };

src/tools/compiletest/src/header.rs

@@ -1115,6 +1115,7 @@ fn expand_variables(mut value: String, config: &Config) -> String {
     const CWD: &str = "{{cwd}}";
     const SRC_BASE: &str = "{{src-base}}";
     const BUILD_BASE: &str = "{{build-base}}";
+    const RUST_SRC_BASE: &str = "{{rust-src-base}}";
     const SYSROOT_BASE: &str = "{{sysroot-base}}";
     const TARGET_LINKER: &str = "{{target-linker}}";
     const TARGET: &str = "{{target}}";
@@ -1144,6 +1145,13 @@ fn expand_variables(mut value: String, config: &Config) -> String {
         value = value.replace(TARGET, &config.target);
     }
 
+    if value.contains(RUST_SRC_BASE) {
+        let src_base = config.sysroot_base.join("lib/rustlib/src/rust");
+        src_base.try_exists().expect(&*format!("{} should exists", src_base.display()));
+        let src_base = src_base.read_link().unwrap_or(src_base);
+        value = value.replace(RUST_SRC_BASE, &src_base.to_string_lossy());
+    }
+
     value
 }
 

src/tools/compiletest/src/runtest.rs

@@ -2294,13 +2294,19 @@ impl<'test> TestCx<'test> {
         }
 
         let base_dir = Path::new("/rustc/FAKE_PREFIX");
-        // Paths into the libstd/libcore
+        // Fake paths into the libstd/libcore
         normalize_path(&base_dir.join("library"), "$SRC_DIR");
         // `ui-fulldeps` tests can show paths to the compiler source when testing macros from
         // `rustc_macros`
         // eg. /home/user/rust/compiler
         normalize_path(&base_dir.join("compiler"), "$COMPILER_DIR");
 
+        // Real paths into the libstd/libcore
+        let rust_src_dir = &self.config.sysroot_base.join("lib/rustlib/src/rust");
+        rust_src_dir.try_exists().expect(&*format!("{} should exists", rust_src_dir.display()));
+        let rust_src_dir = rust_src_dir.read_link().unwrap_or(rust_src_dir.to_path_buf());
+        normalize_path(&rust_src_dir.join("library"), "$SRC_DIR_REAL");
+
         // Paths into the build directory
         let test_build_dir = &self.config.build_base;
         let parent_build_dir = test_build_dir.parent().unwrap().parent().unwrap().parent().unwrap();
@@ -2310,9 +2316,6 @@ impl<'test> TestCx<'test> {
         // eg. /home/user/rust/build
         normalize_path(parent_build_dir, "$BUILD_DIR");
 
-        // Paths into lib directory.
-        normalize_path(&parent_build_dir.parent().unwrap().join("lib"), "$LIB_DIR");
-
         if json {
             // escaped newlines in json strings should be readable
             // in the stderr files. There's no point int being correct,

tests/ui/errors/remap-path-prefix-sysroot.rs

@@ -0,0 +1,24 @@
+//@ revisions: with-remap without-remap
+//@ compile-flags: -g -Ztranslate-remapped-path-to-local-path=yes
+//@ [with-remap]compile-flags: --remap-path-prefix={{rust-src-base}}=remapped
+//@ [with-remap]compile-flags: --remap-path-prefix={{src-base}}=remapped-tests-ui
+//@ [without-remap]compile-flags:
+//@ error-pattern: E0507
+
+// The $SRC_DIR*.rs:LL:COL normalisation doesn't kick in automatically
+// as the remapped revision will not begin with $SRC_DIR_REAL,
+// so we have to do it ourselves.
+//@ normalize-stderr-test: ".rs:\d+:\d+" -> ".rs:LL:COL"
+
+use std::thread;
+struct Worker {
+    thread: thread::JoinHandle<()>,
+}
+
+impl Drop for Worker {
+    fn drop(&mut self) {
+        self.thread.join().unwrap();
+    }
+}
+
+pub fn main(){}

tests/ui/errors/remap-path-prefix-sysroot.with-remap.stderr

@@ -0,0 +1,17 @@
+error[E0507]: cannot move out of `self.thread` which is behind a mutable reference
+  --> remapped-tests-ui/errors/remap-path-prefix-sysroot.rs:LL:COL
+   |
+LL |         self.thread.join().unwrap();
+   |         ^^^^^^^^^^^ ------ `self.thread` moved due to this method call
+   |         |
+   |         move occurs because `self.thread` has type `JoinHandle<()>`, which does not implement the `Copy` trait
+   |
+note: `JoinHandle::<T>::join` takes ownership of the receiver `self`, which moves `self.thread`
+  --> remapped/library/std/src/thread/mod.rs:LL:COL
+   |
+LL |     pub fn join(self) -> Result<T> {
+   |                 ^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0507`.

tests/ui/errors/remap-path-prefix-sysroot.without-remap.stderr

@@ -0,0 +1,17 @@
+error[E0507]: cannot move out of `self.thread` which is behind a mutable reference
+  --> $DIR/remap-path-prefix-sysroot.rs:LL:COL
+   |
+LL |         self.thread.join().unwrap();
+   |         ^^^^^^^^^^^ ------ `self.thread` moved due to this method call
+   |         |
+   |         move occurs because `self.thread` has type `JoinHandle<()>`, which does not implement the `Copy` trait
+   |
+note: `JoinHandle::<T>::join` takes ownership of the receiver `self`, which moves `self.thread`
+  --> $SRC_DIR_REAL/std/src/thread/mod.rs:LL:COL
+   |
+LL |     pub fn join(self) -> Result<T> {
+   |                 ^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0507`.