feat: use python_site_packages_path field when available for installing noarch: python packages, CEP-17

Cargo.toml

@@ -33,7 +33,7 @@ async-compression = { version = "0.4.17", features = [
     "zstd",
 ] }
 async-fd-lock = "0.2.0"
-fs4 = "0.10.0"
+fs4 = "0.11.0"
 async-trait = "0.1.83"
 axum = { version = "0.7.7", default-features = false, features = [
     "tokio",

crates/rattler/src/install/python.rs

@@ -1,9 +1,13 @@
-use rattler_conda_types::{Platform, Version};
-use std::borrow::Cow;
-use std::path::{Path, PathBuf};
+use std::{
+    borrow::Cow,
+    path::{Path, PathBuf},
+};
 
-/// Information required for linking no-arch python packages. The struct contains information about
-/// a specific Python version that is installed in an environment.
+use rattler_conda_types::{PackageRecord, Platform, Version};
+
+/// Information required for linking no-arch python packages. The struct
+/// contains information about a specific Python version that is installed in an
+/// environment.
 #[derive(Debug, Clone)]
 pub struct PythonInfo {
     /// The platform that the python package is installed for
@@ -29,9 +33,26 @@ pub enum PythonInfoError {
 }
 
 impl PythonInfo {
-    /// Build an instance based on the version of the python package and the platform it is
-    /// installed for.
-    pub fn from_version(version: &Version, platform: Platform) -> Result<Self, PythonInfoError> {
+    /// Build an instance based on metadata of the package that represents the
+    /// python interpreter.
+    pub fn from_python_record(
+        record: &PackageRecord,
+        platform: Platform,
+    ) -> Result<Self, PythonInfoError> {
+        Self::from_version(
+            record.version.version(),
+            record.python_site_packages_path.as_deref(),
+            platform,
+        )
+    }
+
+    /// Build an instance based on the version of the python package and the
+    /// platform it is installed for.
+    pub fn from_version(
+        version: &Version,
+        site_packages_path: Option<&str>,
+        platform: Platform,
+    ) -> Result<Self, PythonInfoError> {
         // Determine the major, and minor versions of the version
         let (major, minor) = version
             .as_major_minor()
@@ -45,11 +66,16 @@ impl PythonInfo {
         };
 
         // Find the location of the site packages
-        let site_packages_path = if platform.is_windows() {
-            PathBuf::from("Lib/site-packages")
-        } else {
-            PathBuf::from(format!("lib/python{major}.{minor}/site-packages"))
-        };
+        let site_packages_path = site_packages_path.map_or_else(
+            || {
+                if platform.is_windows() {
+                    PathBuf::from("Lib/site-packages")
+                } else {
+                    PathBuf::from(format!("lib/python{major}.{minor}/site-packages"))
+                }
+            },
+            PathBuf::from,
+        );
 
         // Binary directory
         let bin_dir = if platform.is_windows() {
@@ -89,8 +115,8 @@ impl PythonInfo {
         }
     }
 
-    /// Returns the target location of a file in a noarch python package given its location in its
-    /// package archive.
+    /// Returns the target location of a file in a noarch python package given
+    /// its location in its package archive.
     pub fn get_python_noarch_target_path<'a>(&self, relative_path: &'a Path) -> Cow<'a, Path> {
         if let Ok(rest) = relative_path.strip_prefix("site-packages/") {
             self.site_packages_path.join(rest).into()
@@ -101,8 +127,8 @@ impl PythonInfo {
         }
     }
 
-    /// Returns true if this version of python differs so much that a relink is required for all
-    /// noarch python packages.
+    /// Returns true if this version of python differs so much that a relink is
+    /// required for all noarch python packages.
     pub fn is_relink_required(&self, previous: &PythonInfo) -> bool {
         self.short_version.0 != previous.short_version.0
             || self.short_version.1 != previous.short_version.1

crates/rattler/src/install/transaction.rs

@@ -1,9 +1,9 @@
 use std::collections::HashSet;
 
-use crate::install::python::PythonInfoError;
-use crate::install::PythonInfo;
 use rattler_conda_types::{PackageRecord, Platform};
 
+use crate::install::{python::PythonInfoError, PythonInfo};
+
 /// Error that occurred during creation of a Transaction
 #[derive(Debug, thiserror::Error)]
 pub enum TransactionError {
@@ -31,17 +31,19 @@ pub enum TransactionOperation<Old, New> {
         new: New,
     },
 
-    /// Reinstall a package. This can happen if the Python version changed in the environment, we
-    /// need to relink all noarch python packages in that case.
+    /// Reinstall a package. This can happen if the Python version changed in
+    /// the environment, we need to relink all noarch python packages in
+    /// that case.
     Reinstall(Old),
 
     /// Completely remove a package
     Remove(Old),
 }
 
 impl<Old: AsRef<New>, New> TransactionOperation<Old, New> {
-    /// Returns the record of the package to install for this operation. If this operation does not
-    /// refer to an installable package, `None` is returned.
+    /// Returns the record of the package to install for this operation. If this
+    /// operation does not refer to an installable package, `None` is
+    /// returned.
     pub fn record_to_install(&self) -> Option<&New> {
         match self {
             TransactionOperation::Install(record) => Some(record),
@@ -53,8 +55,9 @@ impl<Old: AsRef<New>, New> TransactionOperation<Old, New> {
 }
 
 impl<Old, New> TransactionOperation<Old, New> {
-    /// Returns the record of the package to remove for this operation. If this operation does not
-    /// refer to an removable package, `None` is returned.
+    /// Returns the record of the package to remove for this operation. If this
+    /// operation does not refer to an removable package, `None` is
+    /// returned.
     pub fn record_to_remove(&self) -> Option<&Old> {
         match self {
             TransactionOperation::Install(_) => None,
@@ -65,25 +68,28 @@ impl<Old, New> TransactionOperation<Old, New> {
     }
 }
 
-/// Describes the operations to perform to bring an environment from one state into another.
+/// Describes the operations to perform to bring an environment from one state
+/// into another.
 #[derive(Debug)]
 pub struct Transaction<Old, New> {
     /// A list of operations to update an environment
     pub operations: Vec<TransactionOperation<Old, New>>,
 
-    /// The python version of the target state, or None if python doesnt exist in the environment.
+    /// The python version of the target state, or None if python doesnt exist
+    /// in the environment.
     pub python_info: Option<PythonInfo>,
 
-    /// The python version of the current state, or None if python didnt exist in the previous
-    /// environment.
+    /// The python version of the current state, or None if python didnt exist
+    /// in the previous environment.
     pub current_python_info: Option<PythonInfo>,
 
     /// The target platform of the transaction
     pub platform: Platform,
 }
 
 impl<Old, New> Transaction<Old, New> {
-    /// Return an iterator over the prefix records of all packages that are going to be removed.
+    /// Return an iterator over the prefix records of all packages that are
+    /// going to be removed.
     pub fn removed_packages(&self) -> impl Iterator<Item = &Old> + '_ {
         self.operations
             .iter()
@@ -97,7 +103,8 @@ impl<Old, New> Transaction<Old, New> {
 }
 
 impl<Old: AsRef<New>, New> Transaction<Old, New> {
-    /// Return an iterator over the prefix records of all packages that are going to be installed.
+    /// Return an iterator over the prefix records of all packages that are
+    /// going to be installed.
     pub fn installed_packages(&self) -> impl Iterator<Item = &New> + '_ {
         self.operations
             .iter()
@@ -111,8 +118,8 @@ impl<Old: AsRef<New>, New> Transaction<Old, New> {
 }
 
 impl<Old: AsRef<PackageRecord>, New: AsRef<PackageRecord>> Transaction<Old, New> {
-    /// Constructs a [`Transaction`] by taking the current situation and diffing that against the
-    /// desired situation.
+    /// Constructs a [`Transaction`] by taking the current situation and diffing
+    /// that against the desired situation.
     pub fn from_current_and_desired<
         CurIter: IntoIterator<Item = Old>,
         NewIter: IntoIterator<Item = New>,
@@ -148,7 +155,8 @@ impl<Old: AsRef<PackageRecord>, New: AsRef<PackageRecord>> Transaction<Old, New>
             .map(|r| r.as_ref().name.clone())
             .collect::<HashSet<_>>();
 
-        // Remove all current packages that are not in desired (but keep order of current)
+        // Remove all current packages that are not in desired (but keep order of
+        // current)
         for record in current_iter {
             if !desired_names.contains(&record.as_ref().name) {
                 operations.push(TransactionOperation::Remove(record));
@@ -158,7 +166,8 @@ impl<Old: AsRef<PackageRecord>, New: AsRef<PackageRecord>> Transaction<Old, New>
         // reverse all removals, last in first out
         operations.reverse();
 
-        // Figure out the operations to perform, but keep the order of the original "desired" iterator
+        // Figure out the operations to perform, but keep the order of the original
+        // "desired" iterator
         for record in desired_iter {
             let name = &record.as_ref().name;
             let old_record = current_map.remove(name);
@@ -190,16 +199,16 @@ impl<Old: AsRef<PackageRecord>, New: AsRef<PackageRecord>> Transaction<Old, New>
     }
 }
 
-/// Determine the version of Python used by a set of packages. Returns `None` if none of the
-/// packages refers to a Python installation.
+/// Determine the version of Python used by a set of packages. Returns `None` if
+/// none of the packages refers to a Python installation.
 fn find_python_info(
     records: impl IntoIterator<Item = impl AsRef<PackageRecord>>,
     platform: Platform,
 ) -> Result<Option<PythonInfo>, PythonInfoError> {
     records
         .into_iter()
         .find(|r| is_python_record(r.as_ref()))
-        .map(|record| PythonInfo::from_version(&record.as_ref().version, platform))
+        .map(|record| PythonInfo::from_python_record(record.as_ref(), platform))
         .map_or(Ok(None), |info| info.map(Some))
 }
 

crates/rattler_conda_types/src/package/index.rs

@@ -1,16 +1,17 @@
 use std::path::Path;
 
-use super::PackageFile;
-use crate::{NoArchType, PackageName, VersionWithSource};
+use rattler_macros::sorted;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, skip_serializing_none, OneOrMany};
 
-use rattler_macros::sorted;
+use super::PackageFile;
+use crate::{NoArchType, PackageName, VersionWithSource};
 
 /// A representation of the `index.json` file found in package archives.
 ///
-/// The `index.json` file contains information about the package build and dependencies of the package.
-/// This data makes up the repodata.json file in the repository.
+/// The `index.json` file contains information about the package build and
+/// dependencies of the package. This data makes up the repodata.json file in
+/// the repository.
 #[serde_as]
 #[sorted]
 #[skip_serializing_none]
@@ -22,7 +23,8 @@ pub struct IndexJson {
     /// The build string of the package.
     pub build: String,
 
-    /// The build number of the package. This is also included in the build string.
+    /// The build number of the package. This is also included in the build
+    /// string.
     pub build_number: u64,
 
     /// The package constraints of the package
@@ -33,8 +35,9 @@ pub struct IndexJson {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub depends: Vec<String>,
 
-    /// Features are a deprecated way to specify different feature sets for the conda solver. This is not
-    /// supported anymore and should not be used. Instead, `mutex` packages should be used to specify
+    /// Features are a deprecated way to specify different feature sets for the
+    /// conda solver. This is not supported anymore and should not be used.
+    /// Instead, `mutex` packages should be used to specify
     /// mutually exclusive features.
     pub features: Option<String>,
 
@@ -47,23 +50,29 @@ pub struct IndexJson {
     /// The lowercase name of the package
     pub name: PackageName,
 
-    /// If this package is independent of architecture this field specifies in what way. See
-    /// [`NoArchType`] for more information.
+    /// If this package is independent of architecture this field specifies in
+    /// what way. See [`NoArchType`] for more information.
     #[serde(skip_serializing_if = "NoArchType::is_none")]
     pub noarch: NoArchType,
 
     /// Optionally, the OS the package is build for.
     pub platform: Option<String>,
 
+    /// Optionally a path within the environment of the site-packages directory.
+    /// This field is only present for python interpreter packages.
+    /// This field was introduced with <https://github.com/conda/ceps/blob/main/cep-17.md>.
+    pub python_site_packages_path: Option<String>,
+
     /// The subdirectory that contains this package
     pub subdir: Option<String>,
 
     /// The timestamp when this package was created
     #[serde_as(as = "Option<crate::utils::serde::Timestamp>")]
     pub timestamp: Option<chrono::DateTime<chrono::Utc>>,
 
-    /// Track features are nowadays only used to downweight packages (ie. give them less priority). To
-    /// that effect, the number of track features is counted (number of commas) and the package is downweighted
+    /// Track features are nowadays only used to downweight packages (ie. give
+    /// them less priority). To that effect, the number of track features is
+    /// counted (number of commas) and the package is downweighted
     /// by the number of track_features.
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     #[serde_as(as = "OneOrMany<_>")]

crates/rattler_conda_types/src/repo_data/mod.rs

@@ -19,15 +19,15 @@ use serde_with::{serde_as, skip_serializing_none, OneOrMany};
 use thiserror::Error;
 use url::Url;
 
-use crate::utils::url::add_trailing_slash;
 use crate::{
     build_spec::BuildNumber,
     package::{IndexJson, RunExportsJson},
-    utils::serde::DeserializeFromStrUnchecked,
-    Channel, NoArchType, PackageName, PackageUrl, Platform, RepoDataRecord, VersionWithSource,
-};
-use crate::{
-    utils::serde::sort_map_alphabetically, MatchSpec, Matches, ParseMatchSpecError, ParseStrictness,
+    utils::{
+        serde::{sort_map_alphabetically, DeserializeFromStrUnchecked},
+        url::add_trailing_slash,
+    },
+    Channel, MatchSpec, Matches, NoArchType, PackageName, PackageUrl, ParseMatchSpecError,
+    ParseStrictness, Platform, RepoDataRecord, VersionWithSource,
 };
 
 /// [`RepoData`] is an index of package binaries available on in a subdirectory
@@ -153,6 +153,11 @@ pub struct PackageRecord {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub purls: Option<BTreeSet<PackageUrl>>,
 
+    /// Optionally a path within the environment of the site-packages directory.
+    /// This field is only present for python interpreter packages.
+    /// This field was introduced with <https://github.com/conda/ceps/blob/main/cep-17.md>.
+    pub python_site_packages_path: Option<String>,
+
     /// Run exports that are specified in the package.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub run_exports: Option<RunExportsJson>,
@@ -302,6 +307,7 @@ impl PackageRecord {
             name,
             noarch: NoArchType::default(),
             platform: None,
+            python_site_packages_path: None,
             sha256: None,
             size: None,
             subdir: Platform::current().to_string(),
@@ -324,10 +330,11 @@ impl PackageRecord {
         topological_sort::sort_topologically(records)
     }
 
-    /// Validate that the given package records are valid w.r.t. 'depends' and 'constrains'.
-    /// This function will return Ok(()) if all records form a valid environment, i.e., all dependencies
-    /// of each package are satisfied by the other packages in the list.
-    /// If there is a dependency that is not satisfied, this function will return an error.
+    /// Validate that the given package records are valid w.r.t. 'depends' and
+    /// 'constrains'. This function will return Ok(()) if all records form a
+    /// valid environment, i.e., all dependencies of each package are
+    /// satisfied by the other packages in the list. If there is a
+    /// dependency that is not satisfied, this function will return an error.
     pub fn validate<T: AsRef<PackageRecord>>(
         records: Vec<T>,
     ) -> Result<(), ValidatePackageRecordsError> {
@@ -488,6 +495,7 @@ impl PackageRecord {
             name: index.name,
             noarch: index.noarch,
             platform: index.platform,
+            python_site_packages_path: index.python_site_packages_path,
             sha256,
             size,
             subdir,