Rollup merge of rust-lang#49193 - davidtwco:issue-29893, r=alexcrichton

Host compiler documentation

Fixes rust-lang#29893. Rust Central Station PR: rust-lang/rust-central-station#40

r? @alexcrichton

src/bootstrap/builder.rs

@@ -321,11 +321,13 @@ impl<'a> Builder<'a> {
                 test::RunMake),
             Kind::Bench => describe!(test::Crate, test::CrateLibrustc),
             Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook,
-                doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon,
-                doc::Reference, doc::Rustdoc, doc::RustByExample, doc::CargoBook),
-            Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts,
-                dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo,
-                dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign),
+                doc::Standalone, doc::Std, doc::Test, doc::WhitelistedRustc, doc::Rustc,
+                doc::ErrorIndex, doc::Nomicon, doc::Reference, doc::Rustdoc, doc::RustByExample,
+                doc::CargoBook),
+            Kind::Dist => describe!(dist::Docs, dist::RustcDocs, dist::Mingw, dist::Rustc,
+                dist::DebuggerScripts, dist::Std, dist::Analysis, dist::Src,
+                dist::PlainSourceTarball, dist::Cargo, dist::Rls, dist::Rustfmt, dist::Extended,
+                dist::HashSign),
             Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls,
                 install::Rustfmt, install::Analysis, install::Src, install::Rustc),
         }

src/bootstrap/dist.rs

@@ -102,7 +102,7 @@ impl Step for Docs {
 
         let dst = image.join("share/doc/rust/html");
         t!(fs::create_dir_all(&dst));
-        let src = build.out.join(host).join("doc");
+        let src = build.doc_out(host);
         cp_r(&src, &dst);
 
         let mut cmd = rust_installer(builder);
@@ -120,14 +120,69 @@ impl Step for Docs {
         build.run(&mut cmd);
         t!(fs::remove_dir_all(&image));
 
-        // As part of this step, *also* copy the docs directory to a directory which
-        // buildbot typically uploads.
-        if host == build.build {
-            let dst = distdir(build).join("doc").join(build.rust_package_vers());
-            t!(fs::create_dir_all(&dst));
-            cp_r(&src, &dst);
+        distdir(build).join(format!("{}-{}.tar.gz", name, host))
+    }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustcDocs {
+    pub stage: u32,
+    pub host: Interned<String>,
+}
+
+impl Step for RustcDocs {
+    type Output = PathBuf;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/librustc")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(RustcDocs {
+            stage: run.builder.top_stage,
+            host: run.target,
+        });
+    }
+
+    /// Builds the `rustc-docs` installer component.
+    fn run(self, builder: &Builder) -> PathBuf {
+        let build = builder.build;
+        let host = self.host;
+
+        let name = pkgname(build, "rustc-docs");
+
+        println!("Dist compiler docs ({})", host);
+        if !build.config.compiler_docs {
+            println!("\tskipping - compiler docs disabled");
+            return distdir(build).join(format!("{}-{}.tar.gz", name, host));
         }
 
+        builder.default_doc(None);
+
+        let image = tmpdir(build).join(format!("{}-{}-image", name, host));
+        let _ = fs::remove_dir_all(&image);
+
+        let dst = image.join("share/doc/rust/html");
+        t!(fs::create_dir_all(&dst));
+        let src = build.compiler_doc_out(host);
+        cp_r(&src, &dst);
+
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+           .arg("--product-name=Rustc-Documentation")
+           .arg("--rel-manifest-dir=rustlib")
+           .arg("--success-message=Rustc-documentation-is-installed.")
+           .arg("--image-dir").arg(&image)
+           .arg("--work-dir").arg(&tmpdir(build))
+           .arg("--output-dir").arg(&distdir(build))
+           .arg(format!("--package-name={}-{}", name, host))
+           .arg("--component-name=rustc-docs")
+           .arg("--legacy-manifest-dirs=rustlib,cargo")
+           .arg("--bulk-dirs=share/doc/rust/html");
+        build.run(&mut cmd);
+        t!(fs::remove_dir_all(&image));
+
         distdir(build).join(format!("{}-{}.tar.gz", name, host))
     }
 }

src/bootstrap/doc.rs

@@ -17,12 +17,13 @@
 //! Everything here is basically just a shim around calling either `rustbook` or
 //! `rustdoc`.
 
+use std::collections::HashSet;
 use std::fs::{self, File};
 use std::io::prelude::*;
 use std::io;
 use std::path::{PathBuf, Path};
 
-use Mode;
+use {Build, Mode};
 use build_helper::up_to_date;
 
 use util::{cp_r, symlink_dir};
@@ -483,21 +484,17 @@ impl Step for Std {
         let mut cargo = builder.cargo(compiler, Mode::Libstd, target, "doc");
         compile::std_cargo(builder, &compiler, target, &mut cargo);
 
-        // We don't want to build docs for internal std dependencies unless
-        // in compiler-docs mode. When not in that mode, we whitelist the crates
-        // for which docs must be built.
-        if !build.config.compiler_docs {
-            cargo.arg("--no-deps");
-            for krate in &["alloc", "core", "std", "std_unicode"] {
-                cargo.arg("-p").arg(krate);
-                // Create all crate output directories first to make sure rustdoc uses
-                // relative links.
-                // FIXME: Cargo should probably do this itself.
-                t!(fs::create_dir_all(out_dir.join(krate)));
-            }
+        // Keep a whitelist so we do not build internal stdlib crates, these will be
+        // build by the rustc step later if enabled.
+        cargo.arg("--no-deps");
+        for krate in &["alloc", "core", "std", "std_unicode"] {
+            cargo.arg("-p").arg(krate);
+            // Create all crate output directories first to make sure rustdoc uses
+            // relative links.
+            // FIXME: Cargo should probably do this itself.
+            t!(fs::create_dir_all(out_dir.join(krate)));
         }
 
-
         build.run(&mut cargo);
         cp_r(&my_out, &out);
     }
@@ -564,12 +561,12 @@ impl Step for Test {
 }
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
-pub struct Rustc {
+pub struct WhitelistedRustc {
     stage: u32,
     target: Interned<String>,
 }
 
-impl Step for Rustc {
+impl Step for WhitelistedRustc {
     type Output = ();
     const DEFAULT: bool = true;
     const ONLY_HOSTS: bool = true;
@@ -580,21 +577,26 @@ impl Step for Rustc {
     }
 
     fn make_run(run: RunConfig) {
-        run.builder.ensure(Rustc {
+        run.builder.ensure(WhitelistedRustc {
             stage: run.builder.top_stage,
             target: run.target,
         });
     }
 
-    /// Generate all compiler documentation.
+    /// Generate whitelisted compiler crate documentation.
     ///
-    /// This will generate all documentation for the compiler libraries and their
-    /// dependencies. This is largely just a wrapper around `cargo doc`.
+    /// This will generate all documentation for crates that are whitelisted
+    /// to be included in the standard documentation. This documentation is
+    /// included in the standard Rust documentation, so we should always
+    /// document it and symlink to merge with the rest of the std and test
+    /// documentation. We don't build other compiler documentation
+    /// here as we want to be able to keep it separate from the standard
+    /// documentation. This is largely just a wrapper around `cargo doc`.
     fn run(self, builder: &Builder) {
         let build = builder.build;
         let stage = self.stage;
         let target = self.target;
-        println!("Documenting stage{} compiler ({})", stage, target);
+        println!("Documenting stage{} whitelisted compiler ({})", stage, target);
         let out = build.doc_out(target);
         t!(fs::create_dir_all(&out));
         let compiler = builder.compiler(stage, build.build);
@@ -620,24 +622,116 @@ impl Step for Rustc {
         let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "doc");
         compile::rustc_cargo(build, &mut cargo);
 
-        if build.config.compiler_docs {
-            // src/rustc/Cargo.toml contains a bin crate called rustc which
-            // would otherwise overwrite the docs for the real rustc lib crate.
-            cargo.arg("-p").arg("rustc_driver");
-        } else {
-            // Like with libstd above if compiler docs aren't enabled then we're not
-            // documenting internal dependencies, so we have a whitelist.
-            cargo.arg("--no-deps");
-            for krate in &["proc_macro"] {
-                cargo.arg("-p").arg(krate);
-            }
+        // We don't want to build docs for internal compiler dependencies in this
+        // step (there is another step for that). Therefore, we whitelist the crates
+        // for which docs must be built.
+        cargo.arg("--no-deps");
+        for krate in &["proc_macro"] {
+            cargo.arg("-p").arg(krate);
         }
 
         build.run(&mut cargo);
         cp_r(&my_out, &out);
     }
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Rustc {
+    stage: u32,
+    target: Interned<String>,
+}
+
+impl Step for Rustc {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        let builder = run.builder;
+        run.krate("rustc-main").default_condition(builder.build.config.docs)
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(Rustc {
+            stage: run.builder.top_stage,
+            target: run.target,
+        });
+    }
+
+    /// Generate compiler documentation.
+    ///
+    /// This will generate all documentation for compiler and dependencies.
+    /// Compiler documentation is distributed separately, so we make sure
+    /// we do not merge it with the other documentation from std, test and
+    /// proc_macros. This is largely just a wrapper around `cargo doc`.
+    fn run(self, builder: &Builder) {
+        let build = builder.build;
+        let stage = self.stage;
+        let target = self.target;
+        println!("Documenting stage{} compiler ({})", stage, target);
+        let out = build.compiler_doc_out(target);
+        t!(fs::create_dir_all(&out));
+        let compiler = builder.compiler(stage, build.build);
+        let rustdoc = builder.rustdoc(compiler.host);
+        let compiler = if build.force_use_stage1(compiler, target) {
+            builder.compiler(1, compiler.host)
+        } else {
+            compiler
+        };
+
+        if !build.config.compiler_docs {
+            println!("\tskipping - compiler docs disabled");
+            return;
+        }
+
+        // Build libstd docs so that we generate relative links
+        builder.ensure(Std { stage, target });
+
+        builder.ensure(compile::Rustc { compiler, target });
+        let out_dir = build.stage_out(compiler, Mode::Librustc)
+                           .join(target).join("doc");
+        // We do not symlink to the same shared folder that already contains std library
+        // documentation from previous steps as we do not want to include that.
+        build.clear_if_dirty(&out, &rustdoc);
+        t!(symlink_dir_force(&out, &out_dir));
+
+        let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "doc");
+        compile::rustc_cargo(build, &mut cargo);
+
+        // Only include compiler crates, no dependencies of those, such as `libc`.
+        cargo.arg("--no-deps");
+
+        // Find dependencies for top level crates.
+        let mut compiler_crates = HashSet::new();
+        for root_crate in &["rustc", "rustc_driver"] {
+            let interned_root_crate = INTERNER.intern_str(root_crate);
+            find_compiler_crates(&build, &interned_root_crate, &mut compiler_crates);
+        }
+
+        for krate in &compiler_crates {
+            cargo.arg("-p").arg(krate);
+        }
+
+        build.run(&mut cargo);
+    }
+}
+
+fn find_compiler_crates(
+    build: &Build,
+    name: &Interned<String>,
+    crates: &mut HashSet<Interned<String>>
+) {
+    // Add current crate.
+    crates.insert(*name);
+
+    // Look for dependencies.
+    for dep in build.crates.get(name).unwrap().deps.iter() {
+        if build.crates.get(dep).unwrap().is_local(build) {
+            find_compiler_crates(build, dep, crates);
+        }
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct ErrorIndex {
     target: Interned<String>,

src/bootstrap/lib.rs

@@ -511,6 +511,11 @@ impl Build {
         self.out.join(&*target).join("doc")
     }
 
+    /// Output directory for all documentation for a target
+    fn compiler_doc_out(&self, target: Interned<String>) -> PathBuf {
+        self.out.join(&*target).join("compiler-doc")
+    }
+
     /// Output directory for some generated md crate documentation for a target (temporary)
     fn md_doc_out(&self, target: Interned<String>) -> Interned<PathBuf> {
         INTERNER.intern_path(self.out.join(&*target).join("md-doc"))

src/ci/docker/dist-x86_64-linux/Dockerfile

@@ -84,7 +84,8 @@ ENV HOSTS=x86_64-unknown-linux-gnu
 ENV RUST_CONFIGURE_ARGS \
       --enable-full-tools \
       --enable-sanitizers \
-      --enable-profiler
+      --enable-profiler \
+      --enable-compiler-docs
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
 
 # This is the only builder which will create source tarballs