Auto merge of #49193 - davidtwco:issue-29893, r=<try>

Host compiler documentation

Fixes #29893. Rust Central Station PR: rust-lang/rust-central-station#40

r? @alexcrichton

src/bootstrap/builder.rs

@@ -316,11 +316,13 @@ impl<'a> Builder<'a> {
                 test::Rustfmt, test::Miri, test::Clippy, test::RustdocJS, test::RustdocTheme),
             Kind::Bench => describe!(test::Crate, test::CrateLibrustc),
             Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook,
-                doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon,
-                doc::Reference, doc::Rustdoc, doc::RustByExample, doc::CargoBook),
-            Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts,
-                dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo,
-                dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign),
+                doc::Standalone, doc::Std, doc::Test, doc::WhitelistedRustc, doc::Rustc,
+                doc::ErrorIndex, doc::Nomicon, doc::Reference, doc::Rustdoc, doc::RustByExample,
+                doc::CargoBook),
+            Kind::Dist => describe!(dist::Docs, dist::RustcDocs, dist::Mingw, dist::Rustc,
+                dist::DebuggerScripts, dist::Std, dist::Analysis, dist::Src,
+                dist::PlainSourceTarball, dist::Cargo, dist::Rls, dist::Rustfmt, dist::Extended,
+                dist::HashSign),
             Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls,
                 install::Rustfmt, install::Analysis, install::Src, install::Rustc),
         }

src/bootstrap/dist.rs

@@ -102,7 +102,7 @@ impl Step for Docs {
 
         let dst = image.join("share/doc/rust/html");
         t!(fs::create_dir_all(&dst));
-        let src = build.out.join(host).join("doc");
+        let src = build.doc_out(host);
         cp_r(&src, &dst);
 
         let mut cmd = rust_installer(builder);
@@ -132,6 +132,77 @@ impl Step for Docs {
     }
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcDocs {
+    pub stage: u32,
+    pub host: Interned<String>,
+}
+
+impl Step for RustcDocs {
+    type Output = PathBuf;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/librustc")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(RustcDocs {
+            stage: run.builder.top_stage,
+            host: run.target,
+        });
+    }
+
+    /// Builds the `rustc-docs` installer component.
+    fn run(self, builder: &Builder) -> PathBuf {
+        let build = builder.build;
+        let host = self.host;
+
+        let name = pkgname(build, "rustc-docs");
+
+        println!("Dist compiler docs ({})", host);
+        if !build.config.compiler_docs {
+            println!("\tskipping - compiler docs disabled");
+            return distdir(build).join(format!("{}-{}.tar.gz", name, host));
+        }
+
+        builder.default_doc(None);
+
+        let image = tmpdir(build).join(format!("{}-{}-image", name, host));
+        let _ = fs::remove_dir_all(&image);
+
+        let dst = image.join("share/doc/rust/html");
+        t!(fs::create_dir_all(&dst));
+        let src = build.compiler_doc_out(host);
+        cp_r(&src, &dst);
+
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+           .arg("--product-name=Rustc-Documentation")
+           .arg("--rel-manifest-dir=rustlib")
+           .arg("--success-message=Rustc-documentation-is-installed.")
+           .arg("--image-dir").arg(&image)
+           .arg("--work-dir").arg(&tmpdir(build))
+           .arg("--output-dir").arg(&distdir(build))
+           .arg(format!("--package-name={}-{}", name, host))
+           .arg("--component-name=rustc-docs")
+           .arg("--legacy-manifest-dirs=rustlib,cargo")
+           .arg("--bulk-dirs=share/doc/rust/html");
+        build.run(&mut cmd);
+        t!(fs::remove_dir_all(&image));
+
+        // As part of this step, *also* copy the docs directory to a directory which
+        // buildbot typically uploads.
+        if host == build.build {
+            let dst = distdir(build).join("rustc-doc").join(build.rust_package_vers());
+            t!(fs::create_dir_all(&dst));
+            cp_r(&src, &dst);
+        }
+
+        distdir(build).join(format!("{}-{}.tar.gz", name, host))
+    }
+}
+
 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
     let mut found = Vec::with_capacity(files.len());
 

src/bootstrap/doc.rs

@@ -483,21 +483,17 @@ impl Step for Std {
         let mut cargo = builder.cargo(compiler, Mode::Libstd, target, "doc");
         compile::std_cargo(builder, &compiler, target, &mut cargo);
 
-        // We don't want to build docs for internal std dependencies unless
-        // in compiler-docs mode. When not in that mode, we whitelist the crates
-        // for which docs must be built.
-        if !build.config.compiler_docs {
-            cargo.arg("--no-deps");
-            for krate in &["alloc", "core", "std", "std_unicode"] {
-                cargo.arg("-p").arg(krate);
-                // Create all crate output directories first to make sure rustdoc uses
-                // relative links.
-                // FIXME: Cargo should probably do this itself.
-                t!(fs::create_dir_all(out_dir.join(krate)));
-            }
+        // Keep a whitelist so we do not build internal stdlib crates, these will be
+        // build by the rustc step later if enabled.
+        cargo.arg("--no-deps");
+        for krate in &["alloc", "core", "std", "std_unicode"] {
+            cargo.arg("-p").arg(krate);
+            // Create all crate output directories first to make sure rustdoc uses
+            // relative links.
+            // FIXME: Cargo should probably do this itself.
+            t!(fs::create_dir_all(out_dir.join(krate)));
         }
 
-
         build.run(&mut cargo);
         cp_r(&my_out, &out);
     }
@@ -563,6 +559,81 @@ impl Step for Test {
     }
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct WhitelistedRustc {
+    stage: u32,
+    target: Interned<String>,
+}
+
+impl Step for WhitelistedRustc {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        let builder = run.builder;
+        run.krate("rustc-main").default_condition(builder.build.config.docs)
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(WhitelistedRustc {
+            stage: run.builder.top_stage,
+            target: run.target,
+        });
+    }
+
+    /// Generate whitelisted compiler crate documentation.
+    ///
+    /// This will generate all documentation for crates that are whitelisted
+    /// to be included in the standard documentation. This documentation is
+    /// included in the standard Rust documentation, so we should always
+    /// document it and symlink to merge with the rest of the std and test
+    /// documentation. We don't build other compiler documentation
+    /// here as we want to be able to keep it separate from the standard
+    /// documentation. This is largely just a wrapper around `cargo doc`.
+    fn run(self, builder: &Builder) {
+        let build = builder.build;
+        let stage = self.stage;
+        let target = self.target;
+        println!("Documenting stage{} whitelisted compiler ({})", stage, target);
+        let out = build.doc_out(target);
+        t!(fs::create_dir_all(&out));
+        let compiler = builder.compiler(stage, build.build);
+        let rustdoc = builder.rustdoc(compiler.host);
+        let compiler = if build.force_use_stage1(compiler, target) {
+            builder.compiler(1, compiler.host)
+        } else {
+            compiler
+        };
+
+        // Build libstd docs so that we generate relative links
+        builder.ensure(Std { stage, target });
+
+        builder.ensure(compile::Rustc { compiler, target });
+        let out_dir = build.stage_out(compiler, Mode::Librustc)
+                           .join(target).join("doc");
+
+        // See docs in std above for why we symlink
+        let my_out = build.crate_doc_out(target);
+        build.clear_if_dirty(&my_out, &rustdoc);
+        t!(symlink_dir_force(&my_out, &out_dir));
+
+        let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "doc");
+        compile::rustc_cargo(build, &mut cargo);
+
+        // We don't want to build docs for internal compiler dependencies in this
+        // step (there is another step for that). Therefore, we whitelist the crates
+        // for which docs must be built.
+        cargo.arg("--no-deps");
+        for krate in &["proc_macro"] {
+            cargo.arg("-p").arg(krate);
+        }
+
+        build.run(&mut cargo);
+        cp_r(&my_out, &out);
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Rustc {
     stage: u32,
@@ -586,16 +657,18 @@ impl Step for Rustc {
         });
     }
 
-    /// Generate all compiler documentation.
+    /// Generate compiler documentation.
     ///
-    /// This will generate all documentation for the compiler libraries and their
-    /// dependencies. This is largely just a wrapper around `cargo doc`.
+    /// This will generate all documentation for compiler and dependencies.
+    /// Compiler documentation is distributed separately, so we make sure
+    /// we do not merge it with the other documentation from std, test and
+    /// proc_macros. This is largely just a wrapper around `cargo doc`.
     fn run(self, builder: &Builder) {
         let build = builder.build;
         let stage = self.stage;
         let target = self.target;
         println!("Documenting stage{} compiler ({})", stage, target);
-        let out = build.doc_out(target);
+        let out = build.compiler_doc_out(target);
         t!(fs::create_dir_all(&out));
         let compiler = builder.compiler(stage, build.build);
         let rustdoc = builder.rustdoc(compiler.host);
@@ -605,6 +678,11 @@ impl Step for Rustc {
             compiler
         };
 
+        if !build.config.compiler_docs {
+            println!("\tskipping - compiler docs disabled");
+            return;
+        }
+
         // Build libstd docs so that we generate relative links
         builder.ensure(Std { stage, target });
 
@@ -613,25 +691,22 @@ impl Step for Rustc {
                            .join(target).join("doc");
 
         // See docs in std above for why we symlink
+        //
+        // This step must happen after other documentation steps. This
+        // invariant ensures that compiler documentation is not included
+        // in the standard documentation tarballs but that all the
+        // documentation from the standard documentation tarballs is included
+        // in the compiler documentation tarball.
         let my_out = build.crate_doc_out(target);
         build.clear_if_dirty(&my_out, &rustdoc);
         t!(symlink_dir_force(&my_out, &out_dir));
 
         let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "doc");
         compile::rustc_cargo(build, &mut cargo);
 
-        if build.config.compiler_docs {
-            // src/rustc/Cargo.toml contains a bin crate called rustc which
-            // would otherwise overwrite the docs for the real rustc lib crate.
-            cargo.arg("-p").arg("rustc_driver");
-        } else {
-            // Like with libstd above if compiler docs aren't enabled then we're not
-            // documenting internal dependencies, so we have a whitelist.
-            cargo.arg("--no-deps");
-            for krate in &["proc_macro"] {
-                cargo.arg("-p").arg(krate);
-            }
-        }
+        // src/rustc/Cargo.toml contains a bin crate called rustc which
+        // would otherwise overwrite the docs for the real rustc lib crate.
+        cargo.arg("-p").arg("rustc_driver");
 
         build.run(&mut cargo);
         cp_r(&my_out, &out);

src/bootstrap/lib.rs

@@ -511,6 +511,11 @@ impl Build {
         self.out.join(&*target).join("doc")
     }
 
+    /// Output directory for all documentation for a target
+    fn compiler_doc_out(&self, target: Interned<String>) -> PathBuf {
+        self.out.join(&*target).join("compiler-doc")
+    }
+
     /// Output directory for some generated md crate documentation for a target (temporary)
     fn md_doc_out(&self, target: Interned<String>) -> Interned<PathBuf> {
         INTERNER.intern_path(self.out.join(&*target).join("md-doc"))

src/ci/docker/dist-x86_64-linux/Dockerfile

@@ -84,7 +84,8 @@ ENV HOSTS=x86_64-unknown-linux-gnu
 ENV RUST_CONFIGURE_ARGS \
       --enable-full-tools \
       --enable-sanitizers \
-      --enable-profiler
+      --enable-profiler \
+      --enable-compiler-docs
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
 
 # This is the only builder which will create source tarballs