Use pip install --report over pip-compile

rye/src/lock.rs

@@ -1,22 +1,21 @@
 use std::collections::{HashMap, HashSet};
 use std::io::{BufWriter, Write};
-use std::path::{Path, PathBuf};
+use std::os::unix::fs::symlink;
+use std::path::Path;
 use std::process::Command;
 use std::sync::Arc;
 use std::{fmt, fs};
 
 use anyhow::{anyhow, bail, Context, Error};
-use once_cell::sync::Lazy;
 use pep508_rs::Requirement;
-use regex::Regex;
+use serde::Deserialize;
 use tempfile::NamedTempFile;
 use url::Url;
 
-use crate::bootstrap::ensure_self_venv;
+use crate::bootstrap::{ensure_self_venv, get_pip_module};
 use crate::pyproject::{normalize_package_name, DependencyKind, PyProject, Workspace};
 use crate::utils::CommandOutput;
 
-static FILE_EDITABLE_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"^-e (file://.*?)\s*$").unwrap());
 static REQUIREMENTS_HEADER: &str = "\
 # generated by rye\n\
 # use `rye lock` or `rye sync` to update this lockfile\
@@ -52,11 +51,33 @@ pub struct LockOptions {
     pub pre: bool,
 }
 
-fn get_pip_compile(output: CommandOutput) -> Result<PathBuf, Error> {
-    let mut pip_compile = ensure_self_venv(output)?;
-    pip_compile.push("bin");
-    pip_compile.push("pip-compile");
-    Ok(pip_compile)
+#[derive(Debug, Deserialize)]
+struct PipReport {
+    install: Vec<PipDependencyReport>,
+}
+
+#[derive(Debug, Deserialize)]
+struct PipMetaData {
+    name: String,
+    version: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct PipDependencyReport {
+    download_info: PipDownloadInfo,
+    metadata: PipMetaData,
+}
+
+#[derive(Debug, Deserialize)]
+struct PipDownloadInfo {
+    url: String,
+    dir_info: Option<PipDirInfo>,
+}
+
+#[derive(Debug, Deserialize)]
+struct PipDirInfo {
+    #[serde(default)]
+    editable: bool,
 }
 
 /// Creates lockfiles for all projects in the workspace.
@@ -111,7 +132,7 @@ pub fn update_workspace_lockfile(
         lockfile,
         lock_options,
         &exclusions,
-        &[],
+        false,
     )?;
     generate_lockfile(
         output,
@@ -120,7 +141,7 @@ pub fn update_workspace_lockfile(
         lockfile,
         lock_options,
         &exclusions,
-        &["--pip-args=--no-deps"],
+        true,
     )?;
 
     Ok(())
@@ -142,6 +163,20 @@ fn find_exclusions(projects: &[PyProject]) -> Result<HashSet<Requirement>, Error
     Ok(rv)
 }
 
+fn copy_constraints(lockfile: &Path, out: &Path) -> Result<(), Error> {
+    let mut out = fs::File::create(out)?;
+    if let Ok(contents) = fs::read_to_string(lockfile) {
+        for line in contents.lines() {
+            let line = line.trim();
+            if line.starts_with('#') || line.starts_with('-') {
+                continue;
+            }
+            writeln!(out, "{}", line)?;
+        }
+    }
+    Ok(())
+}
+
 fn dump_dependencies(
     pyproject: &PyProject,
     local_projects: &HashMap<String, String>,
@@ -205,7 +240,7 @@ pub fn update_single_project_lockfile(
         lockfile,
         lock_options,
         &exclusions,
-        &[],
+        false,
     )?;
 
     Ok(())
@@ -218,49 +253,56 @@ fn generate_lockfile(
     lockfile: &Path,
     lock_options: &LockOptions,
     exclusions: &HashSet<Requirement>,
-    extra_args: &[&str],
+    no_deps: bool,
 ) -> Result<(), Error> {
     let scratch = tempfile::tempdir()?;
-    let requirements_file = scratch.path().join("requirements.txt");
-    if lockfile.is_file() {
-        fs::copy(lockfile, &requirements_file)?;
-    } else {
-        fs::write(lockfile, b"")?;
-    }
+    let constraints_file = scratch.path().join("requirements.txt");
+    let report_file = scratch.path().join("report.json");
+    copy_constraints(lockfile, &constraints_file)?;
+    let self_venv = ensure_self_venv(output)?;
+    symlink(get_pip_module(&self_venv), scratch.path().join("pip"))
+        .context("failed linking pip module into for lock")?;
 
-    let pip_compile_path = get_pip_compile(output)?;
-    let mut cmd = Command::new(pip_compile_path);
-    cmd.arg("--resolver=backtracking")
-        .arg("--no-annotate")
-        .arg("--strip-extras")
-        .arg("--allow-unsafe")
-        .arg("--no-header")
-        .arg("-o")
-        .arg(&requirements_file)
+    // TODO: use a function that returns this.
+    let mut cmd = Command::new(workspace_path.join(".venv/bin/python"));
+    cmd.arg("-mpip")
+        .arg("install")
+        .arg("--ignore-installed")
+        .arg("--dry-run")
+        .arg("--report")
+        .arg(&report_file)
+        .arg("-c")
+        .arg(&constraints_file)
+        .arg("-r")
         .arg(requirements_file_in)
+        .env("PYTHONPATH", scratch.path())
+        .env("PROJECT_ROOT", workspace_path)
         .env("PYTHONWARNINGS", "ignore");
     if output == CommandOutput::Verbose {
         cmd.arg("--verbose");
     } else {
         cmd.arg("-q");
     }
-    for pkg in &lock_options.update {
-        cmd.arg("--upgrade-package");
-        cmd.arg(pkg);
+    if no_deps {
+        cmd.arg("--no-deps");
     }
+    // XXX: this does not work with pip.  Should probably use unearth
+    // for pkg in &lock_options.update {
+    //     cmd.arg("--upgrade-package");
+    //     cmd.arg(pkg);
+    // }
     if lock_options.update_all {
         cmd.arg("--upgrade");
     }
     if lock_options.pre {
         cmd.arg("--pre");
     }
-    cmd.args(extra_args);
     let status = cmd.status().context("unable to run pip-compile")?;
     if !status.success() {
         bail!("failed to generate lockfile");
     };
 
-    finalize_lockfile(&requirements_file, lockfile, workspace_path, exclusions)?;
+    finalize_lockfile(&report_file, lockfile, workspace_path, exclusions)?;
 
     Ok(())
 }
@@ -273,26 +315,50 @@ fn finalize_lockfile(
 ) -> Result<(), Error> {
     let mut rv = BufWriter::new(fs::File::create(out)?);
     writeln!(rv, "{}", REQUIREMENTS_HEADER)?;
-    for line in fs::read_to_string(generated)?.lines() {
-        if let Some(m) = FILE_EDITABLE_RE.captures(line) {
-            let url = Url::parse(&m[1]).context("invalid editable URL generated")?;
+
+    let report: PipReport = serde_json::from_slice(&fs::read(generated)?)?;
+
+    for dep in &report.install {
+        // editable
+        if dep
+            .download_info
+            .dir_info
+            .as_ref()
+            .map_or(false, |x| x.editable)
+        {
+            let url =
+                Url::parse(&dep.download_info.url).context("invalid editable URL generated")?;
             if url.scheme() == "file" {
                 let rel_url = make_relative_url(Path::new(url.path()), workspace_root)?;
                 writeln!(rv, "-e {}", rel_url)?;
                 continue;
             }
-        } else if let Ok(ref req) = line.trim().parse::<Requirement>() {
+        } else {
             // TODO: this does not evaluate markers
             if exclusions.iter().any(|x| {
-                normalize_package_name(&x.name) == normalize_package_name(&req.name)
-                    && (x.version_or_url.is_none() || x.version_or_url == req.version_or_url)
+                normalize_package_name(&x.name) == normalize_package_name(&dep.metadata.name)
+                    && (x.version_or_url.is_none()
+                        || x.version_or_url.as_ref().map_or(false, |x| match x {
+                            pep508_rs::VersionOrUrl::VersionSpecifier(v) => {
+                                if let Ok(ver) = dep.metadata.version.parse() {
+                                    v.contains(&ver)
+                                } else {
+                                    false
+                                }
+                            }
+                            pep508_rs::VersionOrUrl::Url(_) => false,
+                        }))
             }) {
                 // skip exclusions
-                writeln!(rv, "# excluded {}", line)?;
-                continue;
+                writeln!(
+                    rv,
+                    "# excluded {}=={}",
+                    dep.metadata.name, dep.metadata.version
+                )?;
+            } else {
+                writeln!(rv, "{}=={}", dep.metadata.name, dep.metadata.version)?;
             }
         }
-        writeln!(rv, "{}", line)?;
     }
     Ok(())
 }