Add a mode to read docs.rs metadata when running rustdoc

- Add tests for doc runs
- Only give an error when docs.rs feature is set

  This allows distinguishing between 'any doc run' and 'doc run that uses
  docs.rs features'

- Set RUSTC_BOOTSTRAP for doc runs

  This is required for both docs.rs features and the flags passed by
  docsrs_metadata to cargo.

- Add instructions for running a single test to the readme

Cargo.toml

@@ -20,6 +20,7 @@ chrono-humanize = "0.1.1"
 crates-index = "0.16.2"
 crossbeam-utils = "0.5"
 csv = "1.0.2"
+docsrs-metadata = { git = "https://github.com/rust-lang/docs.rs/" }
 dotenv = "0.13"
 failure = "0.1.3"
 flate2 = "1"

local-crates/docs-rs-features/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "docs-rs-features"
+version = "0.1.0"
+authors = ["Joshua Nelson <[email protected]>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+[package.metadata.docs.rs]
+features = ["docs_rs_feature"]
+
+[features]
+docs_rs_feature = []

local-crates/docs-rs-features/src/main.rs

@@ -0,0 +1,6 @@
+#[cfg(feature = "docs_rs_feature")]
+compile_error!("oh no, a hidden regression!");
+
+fn main() {
+    println!("Hello, world!");
+}

src/experiments.rs

@@ -32,6 +32,7 @@ string_enum!(pub enum Mode {
     CheckOnly => "check-only",
     Clippy => "clippy",
     Rustdoc => "rustdoc",
+    RustdocWithDocsrsMetadata => "rustdoc-with-docsrs-metadata",
     UnstableFeatures => "unstable-features",
 });
 

src/runner/graph.rs

@@ -310,6 +310,12 @@ pub(super) fn build_graph(ex: &Experiment, crates: &[Crate], config: &Config) ->
                         },
                         Mode::Rustdoc => TaskStep::Rustdoc {
                             tc: tc.clone(),
+                            docsrs_metadata: false,
+                            quiet,
+                        },
+                        Mode::RustdocWithDocsrsMetadata => TaskStep::Rustdoc {
+                            tc: tc.clone(),
+                            docsrs_metadata: true,
                             quiet,
                         },
                         Mode::UnstableFeatures => TaskStep::UnstableFeatures { tc: tc.clone() },

src/runner/tasks.rs

@@ -56,7 +56,7 @@ pub(super) enum TaskStep {
     BuildOnly { tc: Toolchain, quiet: bool },
     CheckOnly { tc: Toolchain, quiet: bool },
     Clippy { tc: Toolchain, quiet: bool },
-    Rustdoc { tc: Toolchain, quiet: bool },
+    Rustdoc { tc: Toolchain, docsrs_metadata: bool, quiet: bool },
     UnstableFeatures { tc: Toolchain },
 }
 
@@ -70,7 +70,8 @@ impl fmt::Debug for TaskStep {
             TaskStep::BuildOnly { ref tc, quiet } => ("build", quiet, Some(tc)),
             TaskStep::CheckOnly { ref tc, quiet } => ("check", quiet, Some(tc)),
             TaskStep::Clippy { ref tc, quiet } => ("clippy", quiet, Some(tc)),
-            TaskStep::Rustdoc { ref tc, quiet } => ("doc", quiet, Some(tc)),
+            TaskStep::Rustdoc { ref tc, docsrs_metadata: false, quiet } => ("doc", quiet, Some(tc)),
+            TaskStep::Rustdoc { ref tc, docsrs_metadata: true, quiet } => ("doc+docs.rs-metadata", quiet, Some(tc)),
             TaskStep::UnstableFeatures { ref tc } => ("find unstable features on", false, Some(tc)),
         };
 
@@ -184,9 +185,12 @@ impl Task {
                 TaskStep::Clippy { ref tc, quiet } => {
                     ("linting", test::test_clippy_only, tc, quiet)
                 }
-                TaskStep::Rustdoc { ref tc, quiet } => {
+                TaskStep::Rustdoc { ref tc, docsrs_metadata: false, quiet } => {
                     ("documenting", test::test_rustdoc, tc, quiet)
                 }
+                TaskStep::Rustdoc { ref tc, docsrs_metadata: true, quiet } => {
+                    ("documenting with docs.rs metadata", test::test_rustdoc_with_metadata, tc, quiet)
+                }
                 TaskStep::UnstableFeatures { ref tc } => (
                     "checking unstable",
                     crate::runner::unstable_features::find_unstable_features,

src/runner/test.rs

@@ -6,11 +6,12 @@ use crate::runner::tasks::TaskCtx;
 use crate::runner::OverrideResult;
 use cargo_metadata::diagnostic::DiagnosticLevel;
 use cargo_metadata::{Message, Metadata, PackageId};
+use docsrs_metadata::Metadata as DocsrsMetadata;
 use failure::Error;
 use remove_dir_all::remove_dir_all;
 use rustwide::cmd::{CommandError, ProcessLinesActions, SandboxBuilder};
 use rustwide::{Build, PrepareError};
-use std::collections::{BTreeSet, HashSet};
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::convert::TryFrom;
 
 fn failure_reason(err: &Error) -> FailureReason {
@@ -83,6 +84,7 @@ fn run_cargo<DB: WriteResults>(
     args: &[&str],
     check_errors: bool,
     local_packages_id: &HashSet<PackageId>,
+    env: HashMap<&'static str, String>,
 ) -> Fallible<()> {
     let mut rustflags = format!("--cap-lints={}", ctx.experiment.cap_lints.to_str());
     if let Some(ref tc_rustflags) = ctx.toolchain.rustflags {
@@ -151,6 +153,9 @@ fn run_cargo<DB: WriteResults>(
         .env("CARGO_INCREMENTAL", "0")
         .env("RUST_BACKTRACE", "full")
         .env(rustflags_env, rustflags);
+    for (var, data) in env {
+        command = command.env(var, data);
+    }
 
     if check_errors {
         command = command.process_lines(&mut detect_error);
@@ -241,13 +246,15 @@ fn build<DB: WriteResults>(
         &["build", "--frozen", "--message-format=json"],
         true,
         local_packages_id,
+        HashMap::default(),
     )?;
     run_cargo(
         ctx,
         build_env,
         &["test", "--frozen", "--no-run", "--message-format=json"],
         true,
         local_packages_id,
+        HashMap::default(),
     )?;
     Ok(())
 }
@@ -259,6 +266,7 @@ fn test<DB: WriteResults>(ctx: &TaskCtx<DB>, build_env: &Build) -> Fallible<()>
         &["test", "--frozen"],
         false,
         &HashSet::new(),
+        HashMap::default(),
     )
 }
 
@@ -311,6 +319,7 @@ pub(super) fn test_check_only<DB: WriteResults>(
         ],
         true,
         local_packages_id,
+        HashMap::default(),
     ) {
         Ok(TestResult::BuildFail(failure_reason(&err)))
     } else {
@@ -335,6 +344,7 @@ pub(super) fn test_clippy_only<DB: WriteResults>(
         ],
         true,
         local_packages_id,
+        HashMap::default(),
     ) {
         Ok(TestResult::BuildFail(failure_reason(&err)))
     } else {
@@ -359,6 +369,7 @@ pub(super) fn test_rustdoc<DB: WriteResults>(
         ],
         true,
         local_packages_id,
+        HashMap::new(),
     );
 
     // Make sure to remove the built documentation
@@ -371,3 +382,40 @@ pub(super) fn test_rustdoc<DB: WriteResults>(
         Ok(TestResult::TestPass)
     }
 }
+
+pub(super) fn test_rustdoc_with_metadata<DB: WriteResults>(
+    ctx: &TaskCtx<DB>,
+    build_env: &Build,
+    local_packages_id: &HashSet<PackageId>,
+) -> Fallible<TestResult> {
+    let src = build_env.host_source_dir();
+    let metadata = DocsrsMetadata::from_crate_root(src)?;
+    let cargo_args = metadata.cargo_args(
+        &[
+            "--frozen".into(),
+            "--no-deps".into(),
+            "--document-private-items".into(),
+            "--message-format=json".into(),
+        ],
+        &[],
+    );
+    assert_eq!(cargo_args[0], "rustdoc");
+    let cargo_args: Vec<_> = cargo_args.iter().map(|s| s.as_str()).collect();
+    let mut env = metadata.environment_variables();
+    // docsrs-metadata requires a nightly environment, but crater sometimes runs tests on beta and
+    // stable.
+    // TODO: only allow this for the crate currently being built?
+    env.insert("RUSTC_BOOTSTRAP", "1".to_string());
+
+    let res = run_cargo(ctx, build_env, &cargo_args, true, local_packages_id, env);
+
+    // Make sure to remove the built documentation
+    // There is no point in storing it after the build is done
+    remove_dir_all(&build_env.host_target_dir().join("doc"))?;
+
+    if let Err(err) = res {
+        Ok(TestResult::BuildFail(failure_reason(&err)))
+    } else {
+        Ok(TestResult::TestPass)
+    }
+}

src/server/routes/ui/experiments.rs

@@ -44,6 +44,7 @@ impl ExperimentData {
                 Mode::CheckOnly => "cargo check",
                 Mode::Clippy => "cargo clippy",
                 Mode::Rustdoc => "cargo doc",
+                Mode::RustdocWithDocsrsMetadata => "cargo doc (with docs.rs metadata)",
                 Mode::UnstableFeatures => "unstable features",
             },
             assigned_to: experiment.assigned_to.as_ref().map(|a| a.to_string()),

tests/minicrater/README.md

@@ -24,6 +24,11 @@ The runs' output is hidden by default, but you can show it by setting the
 $ MINICRATER_SHOW_OUTPUT=1 cargo test minicrater -- --ignored --test-threads 1
 ```
 
+To run a single test, use `::`:
+```
+$ cargo test minicrater::single_thread_small -- --ignored
+```
+
 ## Adding tests to minicrater
 
 There are two ways to add a test to minicrater:

tests/minicrater/doc/config.toml

@@ -0,0 +1,27 @@
+[server.bot-acl]
+rust-teams = true
+github = ["pietroalbini"]
+
+[server.labels]
+remove = "^S-"
+experiment-queued = "S-waiting-on-crater"
+experiment-completed = "S-waiting-on-review"
+
+[server.distributed]
+chunk-size = 32
+
+[demo-crates]
+crates = []
+github-repos = []
+local-crates = ["build-pass", "docs-rs-features"]
+
+[sandbox]
+memory-limit = "512M"
+build-log-max-size = "2M"
+build-log-max-lines = 1000
+
+[crates]
+
+[github-repos]
+
+[local-crates]

tests/minicrater/doc/downloads.html.context.expected.json

@@ -0,0 +1,30 @@
+{
+  "available_archives": [
+    {
+      "name": "All the crates",
+      "path": "logs-archives/all.tar.gz"
+    },
+    {
+      "name": "error crates",
+      "path": "logs-archives/error.tar.gz"
+    }
+  ],
+  "crates_count": 2,
+  "nav": [
+    {
+      "active": false,
+      "label": "Summary",
+      "url": "index.html"
+    },
+    {
+      "active": false,
+      "label": "Full report",
+      "url": "full.html"
+    },
+    {
+      "active": true,
+      "label": "Downloads",
+      "url": "downloads.html"
+    }
+  ]
+}