feat: only instantiate conda prefix for pypi solve on request (#3029)

This PR change behavior, that only when source distributions (i.e git, non-editable path, or sdist) are encountered, do we actually instantiate the prefix. If you do not have any source dependencies, we instantiate no prefix before doing a PyPI solve. You can further enforce this by adding no-build = true to your [pypi-options]. Also when a previously built sdist is cached, i.e. the built metadata is available, we also skip instantiation.

So we expect a much faster solve for tomls with little source dependencies and heavy environments. This verification can be seen in the comments. This will be the first step though, there are also repositories like rerun-io/rerun that actually do have numerous source dependencies but only require python for the most of them. A further PR could be introduced where we also only instantiate a partial prefix on solve, this does require some user-input, as we need to know which packages need to be available.

Cargo.toml

@@ -16,6 +16,7 @@ repository = "https://github.com/prefix-dev/pixi"
 
 [workspace.dependencies]
 ahash = "0.8.11"
+anyhow = "*"
 assert_matches = "1.5.0"
 async-fd-lock = "0.2.0"
 async-once-cell = "0.5.4"
@@ -99,6 +100,7 @@ toml_edit = "0.22.22"
 tracing = "0.1.41"
 tracing-subscriber = "0.3.19"
 typed-path = "0.10.0"
+uv-build-frontend = { git = "https://github.com/astral-sh/uv", tag = "0.5.6" }
 uv-distribution-filename = { git = "https://github.com/astral-sh/uv", tag = "0.5.6" }
 uv-distribution-types = { git = "https://github.com/astral-sh/uv", tag = "0.5.6" }
 uv-install-wheel = { git = "https://github.com/astral-sh/uv", tag = "0.5.6" }
@@ -206,6 +208,7 @@ performance = ["pixi_allocator"]
 
 [dependencies]
 ahash = { workspace = true }
+anyhow = { workspace = true }
 assert_matches = { workspace = true }
 async-fd-lock = { workspace = true }
 async-once-cell = { workspace = true }
@@ -247,6 +250,7 @@ minijinja = { workspace = true, features = ["builtins"] }
 once_cell = { workspace = true }
 parking_lot = { workspace = true }
 rstest = { workspace = true }
+uv-build-frontend = { workspace = true }
 uv-distribution-filename = { workspace = true }
 uv-distribution-types = { workspace = true }
 uv-install-wheel = { workspace = true }

pixi.toml

@@ -48,7 +48,7 @@ syrupy = ">=4.8.0,<5"
 tomli-w = ">=1.0,<2"
 
 [feature.pytest.tasks]
-test-common-wheels = { cmd = "pytest --numprocesses=auto tests/wheel_tests/", depends-on = [
+test-common-wheels = { cmd = "pytest -s --numprocesses=auto tests/wheel_tests/", depends-on = [
   "build-release",
 ] }
 test-common-wheels-ci = { cmd = "pytest --numprocesses=auto --verbose tests/wheel_tests/" }

src/cli/shell.rs

@@ -8,13 +8,16 @@ use rattler_shell::{
     shell::{CmdExe, PowerShell, Shell, ShellEnum, ShellScript},
 };
 
-use crate::cli::cli_config::{PrefixUpdateConfig, ProjectConfig};
 use crate::lock_file::UpdateMode;
 use crate::{
     activation::CurrentEnvVarBehavior, environment::get_update_lock_file_and_prefix,
     project::virtual_packages::verify_current_platform_has_required_virtual_packages, prompt,
     Project, UpdateLockFileOptions,
 };
+use crate::{
+    cli::cli_config::{PrefixUpdateConfig, ProjectConfig},
+    project::get_activated_environment_variables,
+};
 use pixi_config::{ConfigCliActivation, ConfigCliPrompt};
 
 #[cfg(target_family = "unix")]
@@ -251,15 +254,15 @@ pub async fn execute(args: Args) -> miette::Result<()> {
     .await?;
 
     // Get the environment variables we need to set activate the environment in the shell.
-    let env = project
-        .get_activated_environment_variables(
-            &environment,
-            CurrentEnvVarBehavior::Exclude,
-            Some(&lock_file_data.lock_file),
-            project.config().force_activate(),
-            project.config().experimental_activation_cache_usage(),
-        )
-        .await?;
+    let env = get_activated_environment_variables(
+        project.env_vars(),
+        &environment,
+        CurrentEnvVarBehavior::Exclude,
+        Some(&lock_file_data.lock_file),
+        project.config().force_activate(),
+        project.config().experimental_activation_cache_usage(),
+    )
+    .await?;
 
     tracing::debug!("Pixi environment activation:\n{:?}", env);
 

src/cli/shell_hook.rs

@@ -11,14 +11,14 @@ use rattler_shell::{
 use serde::Serialize;
 use serde_json;
 
-use crate::activation::CurrentEnvVarBehavior;
 use crate::environment::get_update_lock_file_and_prefix;
 use crate::{
     activation::get_activator,
     cli::cli_config::{PrefixUpdateConfig, ProjectConfig},
     project::{Environment, HasProjectRef},
     prompt, Project, UpdateLockFileOptions,
 };
+use crate::{activation::CurrentEnvVarBehavior, project::get_activated_environment_variables};
 
 /// Print the pixi environment activation script.
 ///
@@ -107,16 +107,15 @@ async fn generate_environment_json(
     force_activate: bool,
     experimental_cache: bool,
 ) -> miette::Result<String> {
-    let environment_variables = environment
-        .project()
-        .get_activated_environment_variables(
-            environment,
-            CurrentEnvVarBehavior::Exclude,
-            Some(lock_file),
-            force_activate,
-            experimental_cache,
-        )
-        .await?;
+    let environment_variables = get_activated_environment_variables(
+        environment.project().env_vars(),
+        environment,
+        CurrentEnvVarBehavior::Exclude,
+        Some(lock_file),
+        force_activate,
+        experimental_cache,
+    )
+    .await?;
 
     let shell_env = ShellEnv {
         environment_variables,

src/conda_prefix_updater.rs

@@ -0,0 +1,115 @@
+use crate::build::BuildContext;
+use crate::environment::{self, PythonStatus};
+use crate::lock_file::IoConcurrencyLimit;
+use crate::prefix::Prefix;
+use crate::project::grouped_environment::{GroupedEnvironment, GroupedEnvironmentName};
+use crate::project::HasProjectRef;
+use futures::TryFutureExt;
+use miette::IntoDiagnostic;
+use pixi_manifest::FeaturesExt;
+use pixi_record::PixiRecord;
+use rattler::package_cache::PackageCache;
+use rattler_conda_types::Platform;
+
+/// A struct that contains the result of updating a conda prefix.
+pub struct CondaPrefixUpdated {
+    /// The name of the group that was updated.
+    pub group: GroupedEnvironmentName,
+    /// The prefix that was updated.
+    pub prefix: Prefix,
+    /// Any change to the python interpreter.
+    pub python_status: Box<PythonStatus>,
+}
+
+#[derive(Clone)]
+/// A task that updates the prefix for a given environment.
+pub struct CondaPrefixUpdater<'a> {
+    pub group: GroupedEnvironment<'a>,
+    pub platform: Platform,
+    pub package_cache: PackageCache,
+    pub io_concurrency_limit: IoConcurrencyLimit,
+    pub build_context: BuildContext,
+}
+
+impl<'a> CondaPrefixUpdater<'a> {
+    /// Creates a new prefix task.
+    pub fn new(
+        group: GroupedEnvironment<'a>,
+        platform: Platform,
+        package_cache: PackageCache,
+        io_concurrency_limit: IoConcurrencyLimit,
+        build_context: BuildContext,
+    ) -> Self {
+        Self {
+            group,
+            package_cache,
+            io_concurrency_limit,
+            build_context,
+            platform,
+        }
+    }
+
+    /// Updates the prefix for the given environment.
+    pub(crate) async fn update(
+        &self,
+        pixi_records: Vec<PixiRecord>,
+    ) -> miette::Result<CondaPrefixUpdated> {
+        tracing::debug!(
+            "updating prefix for '{}'",
+            self.group.name().fancy_display()
+        );
+
+        let channels = self
+            .group
+            .channel_urls(&self.group.project().channel_config())
+            .into_diagnostic()?;
+
+        // Spawn a task to determine the currently installed packages.
+        let prefix_clone = self.group.prefix().clone();
+        let installed_packages_future =
+            tokio::task::spawn_blocking(move || prefix_clone.find_installed_packages())
+                .unwrap_or_else(|e| match e.try_into_panic() {
+                    Ok(panic) => std::panic::resume_unwind(panic),
+                    Err(_err) => Err(miette::miette!("the operation was cancelled")),
+                });
+
+        // Wait until the conda records are available and until the installed packages
+        // for this prefix are available.
+        let installed_packages = installed_packages_future.await?;
+
+        let has_existing_packages = !installed_packages.is_empty();
+        let group_name = self.group.name().clone();
+        let client = self.group.project().authenticated_client().clone();
+        let prefix = self.group.prefix();
+
+        let python_status = environment::update_prefix_conda(
+            &prefix,
+            self.package_cache.clone(),
+            client,
+            installed_packages,
+            pixi_records,
+            self.group.virtual_packages(self.platform),
+            channels,
+            self.platform,
+            &format!(
+                "{} conda prefix '{}'",
+                if has_existing_packages {
+                    "updating"
+                } else {
+                    "creating"
+                },
+                group_name.fancy_display()
+            ),
+            "  ",
+            self.io_concurrency_limit.clone().into(),
+            self.build_context.clone(),
+        )
+        .await?;
+
+        Ok(CondaPrefixUpdated {
+            group: group_name,
+            prefix,
+            python_status: Box::new(python_status),
+        })
+    }
+}

src/lib.rs

@@ -2,6 +2,7 @@
 
 pub mod activation;
 pub mod cli;
+pub(crate) mod conda_prefix_updater;
 pub mod diff;
 pub mod environment;
 mod global;
@@ -19,5 +20,6 @@ mod build;
 mod rlimit;
 mod utils;
 
+pub use conda_prefix_updater::{CondaPrefixUpdated, CondaPrefixUpdater};
 pub use lock_file::{load_lock_file, UpdateLockFileOptions};
 pub use project::{DependencyType, Project};

src/lock_file/mod.rs

@@ -7,6 +7,7 @@ mod satisfiability;
 mod update;
 mod utils;
 
+pub use crate::CondaPrefixUpdater;
 use crate::Project;
 use miette::{IntoDiagnostic, WrapErr};
 pub(crate) use package_identifier::PypiPackageIdentifier;
@@ -22,7 +23,7 @@ pub use satisfiability::{
 };
 pub(crate) use update::{LockFileDerivedData, UpdateContext};
 pub use update::{UpdateLockFileOptions, UpdateMode};
-pub(crate) use utils::filter_lock_file;
+pub(crate) use utils::{filter_lock_file, IoConcurrencyLimit};
 
 /// A list of conda packages that are locked for a specific platform.
 pub type LockedCondaPackages = Vec<PixiRecord>;