Auto merge of #543 - jyn514:doc-runs, r=<try>

[WIP] Read docs.rs metadata when running rustdoc

Closes #532

<details><summary>Outdated errors</summary>

I'm having trouble running the test suite, so I haven't yet added a test. My idea was to have a crate that only builds successfully when you read the metadata, but I'm not sure if that would work with the way the test suite is set up.

Current errors:
```
$ cargo test minicrater -- single_thread_small --ignored --test-threads 1
[2020-09-05T02:02:04Z INFO  rustwide::cmd] running `Command { std: "/home/joshua/src/rust/crater/work/cargo-home/bin/cargo" "+stable" "install" "lazy_static", kill_on_drop: false }`
[2020-09-05T02:02:04Z INFO  rustwide::cmd] [stderr]     Updating crates.io index
[2020-09-05T02:02:04Z INFO  rustwide::cmd] [stderr] error: specified package `lazy_static v1.4.0` has no binaries
[2020-09-05T02:02:04Z ERROR crater::utils] Permission denied (os error 13)
[2020-09-05T02:02:04Z ERROR crater::utils] note: run with `RUST_BACKTRACE=1` to display a backtrace.
[2020-09-05T02:02:04Z INFO  crater] command failed
', /home/joshua/.local/lib/cargo/registry/src/jackfan.us.kg-1ecc6299db9ec823/assert_cmd-0.10.1/src/assert.rs:154:13
```

</details>

Before merging, I need to publish the `docsrs-metadata` package to crates.io so crater doesn't depend on a git version.

r? `@pietroalbini` help

Cargo.toml

@@ -20,6 +20,7 @@ chrono-humanize = "0.0.11"
 crates-index = "0.12"
 crossbeam-utils = "0.5"
 csv = "1.0.2"
+docsrs-metadata = { git = "https://github.com/rust-lang/docs.rs/" }
 dotenv = "0.13"
 error-chain = "0.12"
 failure = "0.1.3"

local-crates/docs-rs-features/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "docs-rs-features"
+version = "0.1.0"
+authors = ["Joshua Nelson <[email protected]>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+[package.metadata.docs.rs]
+features = ["docs_rs_feature"]
+
+[features]
+docs_rs_feature = []

local-crates/docs-rs-features/src/main.rs

@@ -0,0 +1,6 @@
+#[cfg(feature = "docs_rs_feature")]
+compile_error!("oh no, a hidden regression!");
+
+fn main() {
+    println!("Hello, world!");
+}

src/runner/test.rs

@@ -6,11 +6,12 @@ use crate::runner::tasks::TaskCtx;
 use crate::runner::OverrideResult;
 use cargo_metadata::diagnostic::DiagnosticLevel;
 use cargo_metadata::{Message, Metadata, PackageId};
+use docsrs_metadata::Metadata as DocsrsMetadata;
 use failure::Error;
 use remove_dir_all::remove_dir_all;
 use rustwide::cmd::{CommandError, ProcessLinesActions, SandboxBuilder};
 use rustwide::{Build, PrepareError};
-use std::collections::{BTreeSet, HashSet};
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::convert::TryFrom;
 
 fn failure_reason(err: &Error) -> FailureReason {
@@ -80,8 +81,9 @@ fn run_cargo<DB: WriteResults>(
     args: &[&str],
     check_errors: bool,
     local_packages_id: &HashSet<PackageId>,
+    mut env: HashMap<&'static str, String>,
 ) -> Fallible<()> {
-    let mut rustflags = format!("--cap-lints={}", ctx.experiment.cap_lints.to_str());
+    let mut rustflags = format!(" --cap-lints={}", ctx.experiment.cap_lints.to_str());
     if let Some(ref tc_rustflags) = ctx.toolchain.rustflags {
         rustflags.push(' ');
         rustflags.push_str(tc_rustflags);
@@ -92,6 +94,7 @@ fn run_cargo<DB: WriteResults>(
     } else {
         "RUSTFLAGS"
     };
+    env.entry(rustflags_env).or_default().push_str(&rustflags);
 
     let mut did_ice = false;
     let mut error_codes = BTreeSet::new();
@@ -146,8 +149,10 @@ fn run_cargo<DB: WriteResults>(
         .cargo()
         .args(args)
         .env("CARGO_INCREMENTAL", "0")
-        .env("RUST_BACKTRACE", "full")
-        .env(rustflags_env, rustflags);
+        .env("RUST_BACKTRACE", "full");
+    for (var, data) in env {
+        command = command.env(var, data);
+    }
 
     if check_errors {
         command = command.process_lines(&mut detect_error);
@@ -238,13 +243,15 @@ fn build<DB: WriteResults>(
         &["build", "--frozen", "--message-format=json"],
         true,
         local_packages_id,
+        HashMap::default(),
     )?;
     run_cargo(
         ctx,
         build_env,
         &["test", "--frozen", "--no-run", "--message-format=json"],
         true,
         local_packages_id,
+        HashMap::default(),
     )?;
     Ok(())
 }
@@ -256,6 +263,7 @@ fn test<DB: WriteResults>(ctx: &TaskCtx<DB>, build_env: &Build) -> Fallible<()>
         &["test", "--frozen"],
         false,
         &HashSet::new(),
+        HashMap::default(),
     )
 }
 
@@ -308,6 +316,7 @@ pub(super) fn test_check_only<DB: WriteResults>(
         ],
         true,
         local_packages_id,
+        HashMap::default(),
     ) {
         Ok(TestResult::BuildFail(failure_reason(&err)))
     } else {
@@ -332,6 +341,7 @@ pub(super) fn test_clippy_only<DB: WriteResults>(
         ],
         true,
         local_packages_id,
+        HashMap::default(),
     ) {
         Ok(TestResult::BuildFail(failure_reason(&err)))
     } else {
@@ -344,18 +354,21 @@ pub(super) fn test_rustdoc<DB: WriteResults>(
     build_env: &Build,
     local_packages_id: &HashSet<PackageId>,
 ) -> Fallible<TestResult> {
+    let src = build_env.host_source_dir();
+    let metadata = DocsrsMetadata::from_crate_root(src)?;
+    let cargo_args = metadata.cargo_args();
+    assert_eq!(cargo_args[0], "doc");
+    let mut cargo_args: Vec<_> = cargo_args.iter().map(|s| s.as_str()).collect();
+    cargo_args.push("--frozen");
+    cargo_args.push("--message-format=json");
+
     let res = run_cargo(
         ctx,
         build_env,
-        &[
-            "doc",
-            "--frozen",
-            "--no-deps",
-            "--document-private-items",
-            "--message-format=json",
-        ],
+        &cargo_args,
         true,
         local_packages_id,
+        metadata.environment_variables(),
     );
 
     // Make sure to remove the built documentation

tests/minicrater/doc/config.toml

@@ -0,0 +1,27 @@
+[server.bot-acl]
+rust-teams = true
+github = ["pietroalbini"]
+
+[server.labels]
+remove = "^S-"
+experiment-queued = "S-waiting-on-crater"
+experiment-completed = "S-waiting-on-review"
+
+[server.distributed]
+chunk-size = 32
+
+[demo-crates]
+crates = []
+github-repos = []
+local-crates = ["build-pass", "docs-rs-features"]
+
+[sandbox]
+memory-limit = "512M"
+build-log-max-size = "2M"
+build-log-max-lines = 1000
+
+[crates]
+
+[github-repos]
+
+[local-crates]

tests/minicrater/doc/downloads.html.context.expected.json

@@ -0,0 +1,30 @@
+{
+  "available_archives": [
+    {
+      "name": "All the crates",
+      "path": "logs-archives/all.tar.gz"
+    },
+    {
+      "name": "error crates",
+      "path": "logs-archives/error.tar.gz"
+    }
+  ],
+  "crates_count": 2,
+  "nav": [
+    {
+      "active": false,
+      "label": "Summary",
+      "url": "index.html"
+    },
+    {
+      "active": false,
+      "label": "Full report",
+      "url": "full.html"
+    },
+    {
+      "active": true,
+      "label": "Downloads",
+      "url": "downloads.html"
+    }
+  ]
+}

tests/minicrater/doc/full.html.context.expected.json

@@ -0,0 +1,76 @@
+{
+  "categories": [
+    [
+      "error",
+      {
+        "Plain": [
+          {
+            "name": "build-pass (local)",
+            "res": "error",
+            "runs": [
+              {
+                "log": "stable/local/build-pass",
+                "res": 0
+              },
+              {
+                "log": "beta/local/build-pass",
+                "res": 0
+              }
+            ],
+            "url": "https://github.com/rust-lang/crater/tree/master/local-crates/build-pass"
+          },
+          {
+            "name": "docs-rs-features (local)",
+            "res": "error",
+            "runs": [
+              {
+                "log": "stable/local/docs-rs-features",
+                "res": 0
+              },
+              {
+                "log": "beta/local/docs-rs-features",
+                "res": 0
+              }
+            ],
+            "url": "https://github.com/rust-lang/crater/tree/master/local-crates/docs-rs-features"
+          }
+        ]
+      }
+    ]
+  ],
+  "comparison_colors": {
+    "error": {
+      "Single": "#d77026"
+    }
+  },
+  "crates_count": 2,
+  "full": true,
+  "info": {
+    "error": 2
+  },
+  "nav": [
+    {
+      "active": false,
+      "label": "Summary",
+      "url": "index.html"
+    },
+    {
+      "active": true,
+      "label": "Full report",
+      "url": "full.html"
+    },
+    {
+      "active": false,
+      "label": "Downloads",
+      "url": "downloads.html"
+    }
+  ],
+  "result_colors": [
+    {
+      "Single": "#d77026"
+    }
+  ],
+  "result_names": [
+    "error"
+  ]
+}

tests/minicrater/doc/index.html.context.expected.json

@@ -0,0 +1,76 @@
+{
+  "categories": [
+    [
+      "error",
+      {
+        "Plain": [
+          {
+            "name": "build-pass (local)",
+            "res": "error",
+            "runs": [
+              {
+                "log": "stable/local/build-pass",
+                "res": 0
+              },
+              {
+                "log": "beta/local/build-pass",
+                "res": 0
+              }
+            ],
+            "url": "https://github.com/rust-lang/crater/tree/master/local-crates/build-pass"
+          },
+          {
+            "name": "docs-rs-features (local)",
+            "res": "error",
+            "runs": [
+              {
+                "log": "stable/local/docs-rs-features",
+                "res": 0
+              },
+              {
+                "log": "beta/local/docs-rs-features",
+                "res": 0
+              }
+            ],
+            "url": "https://github.com/rust-lang/crater/tree/master/local-crates/docs-rs-features"
+          }
+        ]
+      }
+    ]
+  ],
+  "comparison_colors": {
+    "error": {
+      "Single": "#d77026"
+    }
+  },
+  "crates_count": 2,
+  "full": false,
+  "info": {
+    "error": 2
+  },
+  "nav": [
+    {
+      "active": true,
+      "label": "Summary",
+      "url": "index.html"
+    },
+    {
+      "active": false,
+      "label": "Full report",
+      "url": "full.html"
+    },
+    {
+      "active": false,
+      "label": "Downloads",
+      "url": "downloads.html"
+    }
+  ],
+  "result_colors": [
+    {
+      "Single": "#d77026"
+    }
+  ],
+  "result_names": [
+    "error"
+  ]
+}