From 5085d57363ca8e725743e625732f6901eb02841f Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Fri, 30 Oct 2020 11:46:35 -0400 Subject: [PATCH] Revert "Support custom indexes" This reverts commit 78f55ccf48cd6c1560e03566b2286630575e02d8. --- Cargo.lock | 24 +++++++++---------- Cargo.toml | 4 ++-- src/bin/cratesfyi.rs | 30 ++++------------------- src/config.rs | 2 -- src/docbuilder/mod.rs | 2 +- src/docbuilder/queue.rs | 4 ++-- src/docbuilder/rustwide_builder.rs | 25 ++++++-------------- src/index/mod.rs | 38 +++++------------------------- src/lib.rs | 1 - src/test/mod.rs | 5 +--- 10 files changed, 35 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b2609326..63c058655 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "crates-index-diff" -version = "7.1.2" +version = "7.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64af39a9a6805d715f8b72307d70815ed1ee38ef84e9de250fcdd56fe75a0e19" +checksum = "2e6bb290b5bb11353fbb46ca4c68ad2e8f54ab6674e4ee6a94c102054fdaf00f" dependencies = [ "git2", "serde", @@ -1077,9 +1077,9 @@ checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" [[package]] name = "git2" -version = "0.13.12" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6f1a0238d7f8f8fd5ee642f4ebac4dbc03e03d1f78fbe7a3ede35dcf7e2224" +checksum = "11e4b2082980e751c4bf4273e9cbb4a02c655729c8ee8a79f66cad03c8f4d31e" dependencies = [ "bitflags", "libc", @@ -1509,9 +1509,9 @@ checksum = "3286f09f7d4926fc486334f28d8d2e6ebe4f7f9994494b6dab27ddfad2c9b11b" [[package]] name = "libgit2-sys" -version = "0.12.14+1.1.0" +version = "0.12.6+1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f25af58e6495f7caf2919d08f212de550cfa3ed2f5e744988938ea292b9f549" +checksum = "bf81b43f9b45ab07897a780c9b7b26b1504497e469c7a78162fc29e3b8b1c1b3" dependencies = [ "cc", "libc", @@ -1523,9 +1523,9 @@ dependencies = [ [[package]] name = "libssh2-sys" -version = "0.2.19" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca46220853ba1c512fc82826d0834d87b06bcd3c2a42241b7de72f3d2fe17056" +checksum = "eafa907407504b0e683786d4aba47acf250f114d37357d56608333fd167dd0fc" dependencies = [ "cc", "libc", @@ -1537,9 +1537,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.2" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" +checksum = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe" dependencies = [ "cc", "libc", @@ -2872,7 +2872,8 @@ dependencies = [ [[package]] name = "rustwide" version = "0.10.0" -source = "git+https://github.com/rust-lang/rustwide.git?rev=bcf19b84128c983d02257a0ae9b91e5a6772122b#bcf19b84128c983d02257a0ae9b91e5a6772122b" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417d578ebc7fa963bcd06f365f7987c091abeba70eac22dba94b7fd922a95c09" dependencies = [ "base64 0.12.1", "failure", @@ -2880,7 +2881,6 @@ dependencies = [ "fs2", "futures-util", "getrandom", - "git2", "lazy_static", "log 0.4.8", "nix", diff --git a/Cargo.toml b/Cargo.toml index 2277163c0..460b48d4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ log = "0.4" regex = "1" structopt = "0.3" crates-index = "0.15.1" -crates-index-diff = "7.1" +crates-index-diff = "7" reqwest = { version = "0.10.6", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready semver = { version = "0.9", features = ["serde"] } slug = "=0.1.1" @@ -40,7 +40,7 @@ schemamama = "0.3" schemamama_postgres = "0.3" systemstat = "0.1.4" prometheus = { version = "0.10.0", default-features = false } -rustwide = { git = "https://github.com/rust-lang/rustwide.git" , rev = "bcf19b84128c983d02257a0ae9b91e5a6772122b"} +rustwide = "0.10.0" mime_guess = "2" dotenv = "0.15" zstd = "0.5" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 486420167..542d5692b 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -6,8 +6,7 @@ use std::sync::Arc; use docs_rs::db::{self, add_path_into_database, Pool, PoolClient}; use docs_rs::utils::{remove_crate_priority, set_crate_priority}; use docs_rs::{ - BuildQueue, Config, Context, DocBuilder, Index, Metrics, PackageKind, RustwideBuilder, Server, - Storage, + BuildQueue, Config, Context, DocBuilder, Index, Metrics, RustwideBuilder, Server, Storage, }; use failure::{err_msg, Error, ResultExt}; use once_cell::sync::OnceCell; @@ -258,16 +257,8 @@ enum BuildSubcommand { #[structopt(name = "CRATE_VERSION")] crate_version: Option, - /// Url for registry different from cratesio - #[structopt(name = "CRATE_REGISTRY")] - crate_registry: Option, - /// Build a crate at a specific path - #[structopt( - short = "l", - long = "local", - conflicts_with_all(&["CRATE_NAME", "CRATE_VERSION", "CRATE_REGISTRY"]) - )] + #[structopt(short = "l", long = "local", conflicts_with_all(&["CRATE_NAME", "CRATE_VERSION"]))] local: Option, }, @@ -308,7 +299,6 @@ impl BuildSubcommand { Self::Crate { crate_name, crate_version, - crate_registry, local, } => { let mut builder = rustwide_builder()?; @@ -323,10 +313,7 @@ impl BuildSubcommand { &crate_name.ok_or_else(|| err_msg("must specify name if not local"))?, &crate_version .ok_or_else(|| err_msg("must specify version if not local"))?, - crate_registry - .as_ref() - .map(|s| PackageKind::Registry(s.as_str())) - .unwrap_or(PackageKind::CratesIo), + None, ) .context("Building documentation failed")?; } @@ -606,16 +593,7 @@ impl Context for BinContext { fn index(&self) -> Result, Error> { Ok(self .index - .get_or_try_init::<_, Error>(|| { - let config = self.config()?; - Ok(Arc::new( - if let Some(registry_url) = config.registry_url.clone() { - Index::from_url(config.registry_index_path.clone(), registry_url) - } else { - Index::new(config.registry_index_path.clone()) - }?, - )) - })? + .get_or_try_init::<_, Error>(|| Ok(Arc::new(Index::new(&*self.config()?)?)))? .clone()) } } diff --git a/src/config.rs b/src/config.rs index adf75b967..ce57ab219 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,7 +12,6 @@ pub struct Config { pub prefix: PathBuf, pub registry_index_path: PathBuf, - pub registry_url: Option, // Database connection params pub(crate) database_url: String, @@ -57,7 +56,6 @@ impl Config { prefix: prefix.clone(), registry_index_path: env("REGISTRY_INDEX_PATH", prefix.join("crates.io-index"))?, - registry_url: maybe_env("REGISTRY_URL")?, database_url: require_env("CRATESFYI_DATABASE_URL")?, max_pool_size: env("DOCSRS_MAX_POOL_SIZE", 90)?, diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index c5244fa18..568571255 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -4,8 +4,8 @@ mod queue; mod rustwide_builder; pub(crate) use self::limits::Limits; +pub use self::rustwide_builder::RustwideBuilder; pub(crate) use self::rustwide_builder::{BuildResult, DocCoverage}; -pub use self::rustwide_builder::{PackageKind, RustwideBuilder}; use crate::db::Pool; use crate::error::Result; diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index b187fc9f9..5dfb674b0 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -1,6 +1,6 @@ //! Updates registry index and builds new packages -use super::{DocBuilder, PackageKind, RustwideBuilder}; +use super::{DocBuilder, RustwideBuilder}; use crate::error::Result; use crate::utils::get_crate_priority; use crate::Index; @@ -79,7 +79,7 @@ impl DocBuilder { queue.process_next_crate(|krate| { processed = true; - builder.build_package(&krate.name, &krate.version, PackageKind::CratesIo)?; + builder.build_package(&krate.name, &krate.version, None)?; Ok(()) })?; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index ce53f771e..7d6c1929d 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -56,12 +56,6 @@ const ESSENTIAL_FILES_UNVERSIONED: &[&str] = &[ const DUMMY_CRATE_NAME: &str = "empty-library"; const DUMMY_CRATE_VERSION: &str = "1.0.0"; -pub enum PackageKind<'a> { - Local(&'a Path), - CratesIo, - Registry(&'a str), -} - pub struct RustwideBuilder { workspace: Workspace, toolchain: Toolchain, @@ -265,12 +259,7 @@ impl RustwideBuilder { crates_from_path( &self.config.registry_index_path.clone(), &mut |name, version| { - let registry_url = self.config.registry_url.clone(); - let package_kind = registry_url - .as_ref() - .map(|r| PackageKind::Registry(r.as_str())) - .unwrap_or(PackageKind::CratesIo); - if let Err(err) = self.build_package(name, version, package_kind) { + if let Err(err) = self.build_package(name, version, None) { warn!("failed to build package {} {}: {}", name, version, err); } }, @@ -284,14 +273,14 @@ impl RustwideBuilder { err.context(format!("failed to load local package {}", path.display())) })?; let package = metadata.root(); - self.build_package(&package.name, &package.version, PackageKind::Local(path)) + self.build_package(&package.name, &package.version, Some(path)) } pub fn build_package( &mut self, name: &str, version: &str, - kind: PackageKind<'_>, + local: Option<&Path>, ) -> Result { let mut conn = self.db.get()?; @@ -313,10 +302,10 @@ impl RustwideBuilder { let mut build_dir = self.workspace.build_dir(&format!("{}-{}", name, version)); build_dir.purge()?; - let krate = match kind { - PackageKind::Local(path) => Crate::local(path), - PackageKind::CratesIo => Crate::crates_io(name, version), - PackageKind::Registry(registry) => Crate::registry(registry, name, version), + let krate = if let Some(path) = local { + Crate::local(path) + } else { + Crate::crates_io(name, version) }; krate.fetch(&self.workspace)?; diff --git a/src/index/mod.rs b/src/index/mod.rs index 733b80e31..ec415fd49 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -3,7 +3,7 @@ use std::{path::PathBuf, process::Command}; use url::Url; use self::{api::Api, crates::Crates}; -use crate::error::Result; +use crate::{error::Result, Config}; use failure::ResultExt; pub(crate) mod api; @@ -12,10 +12,9 @@ mod crates; pub struct Index { path: PathBuf, api: Api, - repository_url: Option, } -#[derive(Debug, serde::Deserialize, Clone)] +#[derive(serde::Deserialize, Clone)] #[serde(rename_all = "kebab-case")] struct IndexConfig { dl: String, @@ -41,44 +40,19 @@ fn load_config(repo: &git2::Repository) -> Result { } impl Index { - pub fn from_url(path: PathBuf, repository_url: String) -> Result { - let url = repository_url.clone(); - let diff = crates_index_diff::Index::from_path_or_cloned_with_options( - &path, - crates_index_diff::CloneOptions { repository_url }, - ) - .context("initialising registry index repository")?; - - let config = load_config(diff.repository()).context("loading registry config")?; - let api = Api::new(config.api).context("initialising registry api client")?; - Ok(Self { - path, - api, - repository_url: Some(url), - }) - } - - pub fn new(path: PathBuf) -> Result { + pub fn new(app_config: &Config) -> Result { + let path = app_config.registry_index_path.clone(); // This initializes the repository, then closes it afterwards to avoid leaking file descriptors. // See https://github.com/rust-lang/docs.rs/pull/847 let diff = crates_index_diff::Index::from_path_or_cloned(&path) .context("initialising registry index repository")?; let config = load_config(diff.repository()).context("loading registry config")?; let api = Api::new(config.api).context("initialising registry api client")?; - Ok(Self { - path, - api, - repository_url: None, - }) + Ok(Self { path, api }) } pub(crate) fn diff(&self) -> Result { - let options = self - .repository_url - .clone() - .map(|repository_url| crates_index_diff::CloneOptions { repository_url }) - .unwrap_or_default(); - let diff = crates_index_diff::Index::from_path_or_cloned_with_options(&self.path, options) + let diff = crates_index_diff::Index::from_path_or_cloned(&self.path) .context("re-opening registry index for diff")?; Ok(diff) } diff --git a/src/lib.rs b/src/lib.rs index 41adf3e3f..c129cda08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,6 @@ pub use self::build_queue::BuildQueue; pub use self::config::Config; pub use self::context::Context; pub use self::docbuilder::DocBuilder; -pub use self::docbuilder::PackageKind; pub use self::docbuilder::RustwideBuilder; pub use self::index::Index; pub use self::metrics::Metrics; diff --git a/src/test/mod.rs b/src/test/mod.rs index 97c8e25f4..55beee427 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -197,10 +197,7 @@ impl TestEnvironment { pub(crate) fn index(&self) -> Arc { self.index .get_or_init(|| { - Arc::new( - Index::new(self.config().registry_index_path.clone()) - .expect("failed to initialize the index"), - ) + Arc::new(Index::new(&*self.config()).expect("failed to initialize the index")) }) .clone() }