From a6a4b777ece8487c59bce555cbae79852c5d51e6 Mon Sep 17 00:00:00 2001 From: Xin Date: Sat, 9 Mar 2024 16:29:36 +0000 Subject: [PATCH 01/19] Initial implementation of user scheme --- .../src/get_interpreter_info.py | 50 +++-- crates/uv-interpreter/src/interpreter.rs | 172 +++++++++++------- .../uv-interpreter/src/python_environment.rs | 8 + crates/uv/src/commands/pip_install.rs | 8 +- crates/uv/src/main.rs | 9 + 5 files changed, 170 insertions(+), 77 deletions(-) diff --git a/crates/uv-interpreter/src/get_interpreter_info.py b/crates/uv-interpreter/src/get_interpreter_info.py index 4d4b4f2b418a..0c5551ee0412 100644 --- a/crates/uv-interpreter/src/get_interpreter_info.py +++ b/crates/uv-interpreter/src/get_interpreter_info.py @@ -6,7 +6,6 @@ 1: General failure 3: Python version 3 or newer is required """ - import json import os import platform @@ -25,7 +24,6 @@ def format_full_version(info): if sys.version_info[0] < 3: sys.exit(3) - if hasattr(sys, "implementation"): implementation_version = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name @@ -204,7 +202,7 @@ def expand_path(path: str) -> str: } -def get_scheme(): +def get_scheme(user: bool = False): """Return the Scheme for the current interpreter. The paths returned should be absolute. @@ -213,7 +211,7 @@ def get_scheme(): https://github.com/pypa/pip/blob/ae5fff36b0aad6e5e0037884927eaa29163c0611/src/pip/_internal/locations/__init__.py#L230 """ - def get_sysconfig_scheme(): + def get_sysconfig_scheme(user: bool = False): """Get the "scheme" corresponding to the input parameters. Uses the `sysconfig` module to get the scheme. @@ -257,11 +255,23 @@ def _should_use_osx_framework_prefix() -> bool: or our own, and we deal with this special case in ``get_scheme()`` instead. """ return ( - "osx_framework_library" in _AVAILABLE_SCHEMES - and not running_under_virtualenv() - and is_osx_framework() + "osx_framework_library" in _AVAILABLE_SCHEMES + and not running_under_virtualenv() + and is_osx_framework() ) + def _infer_user() -> str: + """Try to find a user scheme for the current platform.""" + if _PREFERRED_SCHEME_API: + return _PREFERRED_SCHEME_API("user") + if is_osx_framework() and not running_under_virtualenv(): + suffixed = "osx_framework_user" + else: + suffixed = f"{os.name}_user" + if suffixed in _AVAILABLE_SCHEMES: + return suffixed + return "posix_user" + def _infer_prefix() -> str: """Try to find a prefix scheme for the current platform. @@ -288,11 +298,15 @@ def _infer_prefix() -> str: suffixed = f"{os.name}_prefix" if suffixed in _AVAILABLE_SCHEMES: return suffixed - if os.name in _AVAILABLE_SCHEMES: # On Windows, prefx is just called "nt". + if os.name in _AVAILABLE_SCHEMES: # On Windows, prefix is just called "nt". return os.name return "posix_prefix" - scheme_name = _infer_prefix() + if user: + scheme_name = _infer_user() + else: + scheme_name = _infer_prefix() + paths = sysconfig.get_paths(scheme=scheme_name) # Logic here is very arbitrary, we're doing it for compatibility, don't ask. @@ -309,7 +323,7 @@ def _infer_prefix() -> str: "data": paths["data"], } - def get_distutils_scheme(): + def get_distutils_scheme(user: bool = False): """Get the "scheme" corresponding to the input parameters. Uses the deprecated `distutils` module to get the scheme. @@ -335,6 +349,7 @@ def get_distutils_scheme(): warnings.simplefilter("ignore") i = d.get_command_obj("install", create=True) + i.user = user i.finalize_options() scheme = {} @@ -350,8 +365,12 @@ def get_distutils_scheme(): scheme.update({"purelib": i.install_lib, "platlib": i.install_lib}) if running_under_virtualenv(): + if user: + prefix = i.install_userbase + else: + prefix = i.prefix scheme["headers"] = os.path.join( - i.prefix, + prefix, "include", "site", f"python{get_major_minor_version()}", @@ -375,10 +394,13 @@ def get_distutils_scheme(): ) if use_sysconfig: - return get_sysconfig_scheme() + return get_sysconfig_scheme(user) else: - return get_distutils_scheme() + return get_distutils_scheme(user) + +# Read the environment variable `_UV_USE_USER_SCHEME` to determine if we should use the user scheme. +user = os.getenv("_UV_USE_USER_SCHEME", "False").upper() in {"TRUE", "1"} markers = { "implementation_name": implementation_name, @@ -401,7 +423,7 @@ def get_distutils_scheme(): "base_executable": getattr(sys, "_base_executable", None), "sys_executable": sys.executable, "stdlib": sysconfig.get_path("stdlib"), - "scheme": get_scheme(), + "scheme": get_scheme(user), "virtualenv": get_virtualenv(), } print(json.dumps(interpreter_info)) diff --git a/crates/uv-interpreter/src/interpreter.rs b/crates/uv-interpreter/src/interpreter.rs index 7f2bdb772bd6..ef1b890654f7 100644 --- a/crates/uv-interpreter/src/interpreter.rs +++ b/crates/uv-interpreter/src/interpreter.rs @@ -1,6 +1,6 @@ use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Output}; use configparser::ini::Ini; use fs_err as fs; @@ -104,6 +104,15 @@ impl Interpreter { } } + /// Return a new [`Interpreter`] with user scheme. + pub fn with_user_scheme(self) -> Result { + let info = InterpreterInfo::query_user_scheme_info(self.sys_executable())?; + Ok(Self { + scheme: info.scheme, + ..self + }) + } + /// Find the best available Python interpreter to use. /// /// If no Python version is provided, we will use the first available interpreter. @@ -114,7 +123,7 @@ impl Interpreter { /// the first available version. /// /// See [`Self::find_version`] for details on the precedence of Python lookup locations. - #[instrument(skip_all, fields(?python_version))] + #[instrument(skip_all, fields(? python_version))] pub fn find_best( python_version: Option<&PythonVersion>, platform: &Platform, @@ -450,67 +459,8 @@ impl InterpreterInfo { /// Return the resolved [`InterpreterInfo`] for the given Python executable. pub(crate) fn query(interpreter: &Path) -> Result { let script = include_str!("get_interpreter_info.py"); - let output = if cfg!(windows) - && interpreter - .extension() - .is_some_and(|extension| extension == "bat") - { - // Multiline arguments aren't well-supported in batch files and `pyenv-win`, for example, trips over it. - // We work around this batch limitation by passing the script via stdin instead. - // This is somewhat more expensive because we have to spawn a new thread to write the - // stdin to avoid deadlocks in case the child process waits for the parent to read stdout. - // The performance overhead is the reason why we only applies this to batch files. - // https://github.com/pyenv-win/pyenv-win/issues/589 - let mut child = Command::new(interpreter) - .arg("-") - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .spawn() - .map_err(|err| Error::PythonSubcommandLaunch { - interpreter: interpreter.to_path_buf(), - err, - })?; - - let mut stdin = child.stdin.take().unwrap(); - - // From the Rust documentation: - // If the child process fills its stdout buffer, it may end up - // waiting until the parent reads the stdout, and not be able to - // read stdin in the meantime, causing a deadlock. - // Writing from another thread ensures that stdout is being read - // at the same time, avoiding the problem. - std::thread::spawn(move || { - stdin - .write_all(script.as_bytes()) - .expect("failed to write to stdin"); - }); - - child.wait_with_output() - } else { - Command::new(interpreter).arg("-c").arg(script).output() - } - .map_err(|err| Error::PythonSubcommandLaunch { - interpreter: interpreter.to_path_buf(), - err, - })?; - // stderr isn't technically a criterion for success, but i don't know of any cases where there - // should be stderr output and if there is, we want to know - if !output.status.success() || !output.stderr.is_empty() { - if output.status.code() == Some(3) { - return Err(Error::Python2OrOlder); - } - - return Err(Error::PythonSubcommandOutput { - message: format!( - "Querying Python at `{}` failed with status {}", - interpreter.display(), - output.status, - ), - stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), - stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), - }); - } + let output = Self::execute_script(interpreter, script, None)?; let data: Self = serde_json::from_slice(&output.stdout).map_err(|err| { Error::PythonSubcommandOutput { @@ -599,6 +549,104 @@ impl InterpreterInfo { Ok(info) } + + /// Return the [`InterpreterInfo`] for the given Python interpreter with user scheme. + pub(crate) fn query_user_scheme_info(interpreter: &Path) -> Result { + let script = include_str!("get_interpreter_info.py"); + let envs = vec![("_UV_USE_USER_SCHEME", "1")]; + let output = Self::execute_script(interpreter, script, Some(envs))?; + + let data: Self = serde_json::from_slice(&output.stdout).map_err(|err| { + Error::PythonSubcommandOutput { + message: format!( + "Querying Python at `{}` did not return the expected data: {err}", + interpreter.display(), + ), + stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), + stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), + } + })?; + + Ok(data) + } + + /// Execute the given Python script using an interpreter. + fn execute_script( + interpreter: &Path, + script: &str, + envs: Option>, + ) -> Result { + let mut command = Command::new(interpreter); + + if let Some(env_variables) = envs { + command.envs(env_variables); + } + + let script_clone = script.to_string(); + let output = if cfg!(windows) + && interpreter + .extension() + .is_some_and(|extension| extension == "bat") + { + // Multiline arguments aren't well-supported in batch files and `pyenv-win`, for example, trips over it. + // We work around this batch limitation by passing the script via stdin instead. + // This is somewhat more expensive because we have to spawn a new thread to write the + // stdin to avoid deadlocks in case the child process waits for the parent to read stdout. + // The performance overhead is the reason why we only applies this to batch files. + // https://github.com/pyenv-win/pyenv-win/issues/589 + let mut child = command + .arg("-") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .map_err(|err| Error::PythonSubcommandLaunch { + interpreter: interpreter.to_path_buf(), + err, + })?; + + let mut stdin = child.stdin.take().unwrap(); + + // From the Rust documentation: + // If the child process fills its stdout buffer, it may end up + // waiting until the parent reads the stdout, and not be able to + // read stdin in the meantime, causing a deadlock. + // Writing from another thread ensures that stdout is being read + // at the same time, avoiding the problem. + std::thread::spawn(move || { + stdin + .write_all(script_clone.as_bytes()) + .expect("failed to write to stdin"); + }); + + child.wait_with_output() + } else { + command.arg("-c").arg(script_clone).output() + } + .map_err(|err| Error::PythonSubcommandLaunch { + interpreter: interpreter.to_path_buf(), + err, + })?; + + // stderr isn't technically a criterion for success, but i don't know of any cases where there + // should be stderr output and if there is, we want to know + if !output.status.success() || !output.stderr.is_empty() { + if output.status.code() == Some(3) { + return Err(Error::Python2OrOlder); + } + + return Err(Error::PythonSubcommandOutput { + message: format!( + "Querying Python at `{}` failed with status {}", + interpreter.display(), + output.status, + ), + stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), + stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), + }); + } + + Ok(output) + } } #[cfg(unix)] diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index 3987b0fa79c2..ff87e6baabfc 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -18,6 +18,14 @@ pub struct PythonEnvironment { } impl PythonEnvironment { + /// Create a [`PythonEnvironment`] from an existing interpreter with user scheme. + pub fn from_interpreter_with_user_scheme(interpreter: Interpreter) -> Result { + Ok(Self { + root: interpreter.prefix().to_path_buf(), + interpreter: interpreter.with_user_scheme()?, + }) + } + /// Create a [`PythonEnvironment`] for an existing virtual environment. pub fn from_virtualenv(platform: Platform, cache: &Cache) -> Result { let Some(venv) = detect_virtual_env()? else { diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 7230be9498bb..9ab1b9e65ea8 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -65,6 +65,7 @@ pub(crate) async fn pip_install( exclude_newer: Option>, python: Option, system: bool, + user: bool, cache: Cache, mut printer: Printer, ) -> Result { @@ -107,13 +108,18 @@ pub(crate) async fn pip_install( // Detect the current Python interpreter. let platform = Platform::current()?; - let venv = if let Some(python) = python.as_ref() { + let detected_env = if let Some(python) = python.as_ref() { PythonEnvironment::from_requested_python(python, &platform, &cache)? } else if system { PythonEnvironment::from_default_python(&platform, &cache)? } else { PythonEnvironment::from_virtualenv(platform, &cache)? }; + + let venv = if user { + PythonEnvironment::from_interpreter_with_user_scheme(detected_env.interpreter().clone())? + } else { detected_env }; + debug!( "Using Python {} environment at {}", venv.interpreter().python_version(), diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index a20438a5f43b..56520869d4a1 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -752,6 +752,14 @@ struct PipInstallArgs { #[clap(long, conflicts_with = "python")] system: bool, + /// Install packages into user site-packages directory + /// + /// The `--user` option allows `uv` to install packages to a location that is specific to a + /// user. The user scheme can be customized by setting the `PYTHONUSERBASE` environment + /// variable. + #[clap(long, conflicts_with = "system")] + user: bool, + /// Use legacy `setuptools` behavior when building source distributions without a /// `pyproject.toml`. #[clap(long)] @@ -1385,6 +1393,7 @@ async fn run() -> Result { args.exclude_newer, args.python, args.system, + args.user, cache, printer, ) From 9c35a6a1490c1b9e2966e2345c33b6752a36b92c Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 00:07:50 +0000 Subject: [PATCH 02/19] Ensure venv scheme paths exist --- .../uv-interpreter/src/python_environment.rs | 19 ++++++++++++++++++- crates/uv/src/commands/pip_install.rs | 5 +++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index ff87e6baabfc..239412ffc9b3 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -1,5 +1,5 @@ -use std::env; use std::path::{Path, PathBuf}; +use std::{env, fs}; use tracing::debug; @@ -121,6 +121,23 @@ impl PythonEnvironment { ) } } + + /// Ensure interpreter scheme directories exist. + /// Model for the scheme in pip: + pub fn ensure_scheme_directories_exist(&self) -> Result<(), Error> { + let directories = vec![ + self.interpreter.platlib(), + self.interpreter.purelib(), + self.interpreter.scripts(), + self.interpreter.data(), + ]; + for path in directories { + if !Path::new(path).exists() { + fs::create_dir_all(path)?; + }; + } + Ok(()) + } } /// Locate the current virtual environment. diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 9ab1b9e65ea8..2beaabfc4079 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -108,6 +108,7 @@ pub(crate) async fn pip_install( // Detect the current Python interpreter. let platform = Platform::current()?; + let detected_env = if let Some(python) = python.as_ref() { PythonEnvironment::from_requested_python(python, &platform, &cache)? } else if system { @@ -120,6 +121,10 @@ pub(crate) async fn pip_install( PythonEnvironment::from_interpreter_with_user_scheme(detected_env.interpreter().clone())? } else { detected_env }; + if user { + venv.ensure_scheme_directories_exist()?; + } + debug!( "Using Python {} environment at {}", venv.interpreter().python_version(), From 425f4f734c78579e875053fe10242184d3301390 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 11:28:55 +0000 Subject: [PATCH 03/19] Fix merge issues --- crates/uv-interpreter/src/interpreter.rs | 20 ++++++++++++++++++++ crates/uv/src/main.rs | 10 ++++++++++ 2 files changed, 30 insertions(+) diff --git a/crates/uv-interpreter/src/interpreter.rs b/crates/uv-interpreter/src/interpreter.rs index 262879bcd8fe..a670247d1170 100644 --- a/crates/uv-interpreter/src/interpreter.rs +++ b/crates/uv-interpreter/src/interpreter.rs @@ -563,6 +563,7 @@ impl InterpreterInfo { "Querying Python at `{}` did not return the expected data: {err}", interpreter.display(), ), + exit_code: output.status, stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), } @@ -628,6 +629,25 @@ impl InterpreterInfo { err, })?; + // stderr isn't technically a criterion for success, but i don't know of any cases where there + // should be stderr output and if there is, we want to know + if !output.status.success() || !output.stderr.is_empty() { + if output.status.code() == Some(3) { + return Err(Error::Python2OrOlder); + } + + return Err(Error::PythonSubcommandOutput { + message: format!( + "Querying Python at `{}` failed with status {}", + interpreter.display(), + output.status, + ), + exit_code: output.status, + stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), + stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), + }); + } + Ok(output) } } diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index f9b97fcb9fa7..ec51aa37b1c2 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -788,6 +788,15 @@ struct PipInstallArgs { #[clap(long, conflicts_with = "python", group = "discovery")] system: bool, + /// Allow `uv` to modify an `EXTERNALLY-MANAGED` Python installation. + /// + /// WARNING: `--break-system-packages` is intended for use in continuous integration (CI) + /// environments, when installing into Python installations that are managed by an external + /// package manager, like `apt`. It should be used with caution, as such Python installations + /// explicitly recommend against modifications by other package managers (like `uv` or `pip`). + #[clap(long, requires = "discovery")] + break_system_packages: bool, + /// Install packages into user site-packages directory /// /// The `--user` option allows `uv` to install packages to a location that is specific to a @@ -1531,6 +1540,7 @@ async fn run() -> Result { args.python, args.system, args.break_system_packages, + args.user, cache, printer, ) From 5cef5d80be7c333abd85838ae3b49829b1dff069 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 16:49:42 +0000 Subject: [PATCH 04/19] Add include_system_site_packages to PyVenvConfiguration --- crates/uv-interpreter/src/cfg.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/uv-interpreter/src/cfg.rs b/crates/uv-interpreter/src/cfg.rs index 0cf15a96e379..50b802f6e92c 100644 --- a/crates/uv-interpreter/src/cfg.rs +++ b/crates/uv-interpreter/src/cfg.rs @@ -10,6 +10,8 @@ pub struct PyVenvConfiguration { pub(crate) virtualenv: bool, /// The version of the `uv` package used to create the virtual environment, if any. pub(crate) uv: bool, + /// If the virtual environment has access to the system packages, per PEP 405. + pub(crate) include_system_site_packages: bool, } impl PyVenvConfiguration { @@ -17,13 +19,14 @@ impl PyVenvConfiguration { pub fn parse(cfg: impl AsRef) -> Result { let mut virtualenv = false; let mut uv = false; + let mut include_system_site_packages = false; // Per https://snarky.ca/how-virtual-environments-work/, the `pyvenv.cfg` file is not a // valid INI file, and is instead expected to be parsed by partitioning each line on the // first equals sign. let content = fs::read_to_string(&cfg)?; for line in content.lines() { - let Some((key, _value)) = line.split_once('=') else { + let Some((key, value)) = line.split_once('=') else { continue; }; match key.trim() { @@ -33,11 +36,14 @@ impl PyVenvConfiguration { "uv" => { uv = true; } + "include-system-site-packages" => { + include_system_site_packages = value.trim() == "true" + } _ => {} } } - Ok(Self { virtualenv, uv }) + Ok(Self { virtualenv, uv, include_system_site_packages }) } /// Returns true if the virtual environment was created with the `virtualenv` package. @@ -49,6 +55,9 @@ impl PyVenvConfiguration { pub fn is_uv(&self) -> bool { self.uv } + + /// Return true if the virtual environment has access to system site packages. + pub fn include_system_site_packages(&self) -> bool { self.include_system_site_packages } } #[derive(Debug, Error)] From bbd910dde689c725c7f54f73c334af44ac103f71 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 16:51:28 +0000 Subject: [PATCH 05/19] Refactor to from_user_scheme * to use requested python, virtualenv and default python in order * ensure the scheme directories exist --- .../uv-interpreter/src/python_environment.rs | 63 +++++++++++-------- crates/uv/src/commands/pip_install.rs | 25 +++++--- 2 files changed, 53 insertions(+), 35 deletions(-) diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index 273977400638..c1ba6d9391c5 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -18,14 +18,6 @@ pub struct PythonEnvironment { } impl PythonEnvironment { - /// Create a [`PythonEnvironment`] from an existing interpreter with user scheme. - pub fn from_interpreter_with_user_scheme(interpreter: Interpreter) -> Result { - Ok(Self { - root: interpreter.prefix().to_path_buf(), - interpreter: interpreter.with_user_scheme()?, - }) - } - /// Create a [`PythonEnvironment`] for an existing virtual environment. pub fn from_virtualenv(platform: Platform, cache: &Cache) -> Result { let Some(venv) = detect_virtual_env()? else { @@ -80,6 +72,44 @@ impl PythonEnvironment { } } + /// Create a [`PythonEnvironment`] with user scheme. + pub fn from_user_scheme(python: Option<&String>, platform: Platform, cache: &Cache) -> Result { + // Attempt to determine the interpreter based on the provided criteria + let interpreter = if let Some(requested_python) = python { + // If a specific Python version is requested + Self::from_requested_python(requested_python, &platform, cache)?.interpreter + } else if let Some(_venv) = detect_virtual_env()? { + // If a virtual environment is detected + Self::from_virtualenv(platform, cache)?.interpreter + } else { + // Fallback to the default Python interpreter + Self::from_default_python(&platform, cache)?.interpreter + }; + + // Apply the user scheme to the determined interpreter + let interpreter_with_user_scheme = interpreter.with_user_scheme()?; + + // Ensure interpreter scheme directories exist, as per the model in pip + // + let directories = vec![ + interpreter_with_user_scheme.platlib(), + interpreter_with_user_scheme.purelib(), + interpreter_with_user_scheme.scripts(), + interpreter_with_user_scheme.data(), + ]; + + for path in directories { + if !Path::new(path).exists() { + fs::create_dir_all(path)?; + } + } + + Ok(Self { + root: interpreter_with_user_scheme.prefix().to_path_buf(), + interpreter: interpreter_with_user_scheme, + }) + } + /// Returns the location of the Python interpreter. pub fn root(&self) -> &Path { &self.root @@ -124,23 +154,6 @@ impl PythonEnvironment { ) } } - - /// Ensure interpreter scheme directories exist. - /// Model for the scheme in pip: - pub fn ensure_scheme_directories_exist(&self) -> Result<(), Error> { - let directories = vec![ - self.interpreter.platlib(), - self.interpreter.purelib(), - self.interpreter.scripts(), - self.interpreter.data(), - ]; - for path in directories { - if !Path::new(path).exists() { - fs::create_dir_all(path)?; - }; - } - Ok(()) - } } /// Locate the current virtual environment. diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 9384d3a1c58f..7e13d9bdc440 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -111,8 +111,9 @@ pub(crate) async fn pip_install( // Detect the current Python interpreter. let platform = Platform::current()?; - - let detected_env = if let Some(python) = python.as_ref() { + let venv = if user { + PythonEnvironment::from_user_scheme(python.as_ref(), platform, &cache)? + } else if let Some(python) = python.as_ref() { PythonEnvironment::from_requested_python(python, &platform, &cache)? } else if system { PythonEnvironment::from_default_python(&platform, &cache)? @@ -120,14 +121,6 @@ pub(crate) async fn pip_install( PythonEnvironment::from_virtualenv(platform, &cache)? }; - let venv = if user { - PythonEnvironment::from_interpreter_with_user_scheme(detected_env.interpreter().clone())? - } else { detected_env }; - - if user { - venv.ensure_scheme_directories_exist()?; - } - debug!( "Using Python {} environment at {}", venv.interpreter().python_version(), @@ -154,6 +147,18 @@ pub(crate) async fn pip_install( } } + // Check if virtualenv has access to the system site-packages + // + if user + && venv.interpreter().is_virtualenv() + && !venv.cfg().unwrap().include_system_site_packages() + { + return Err(anyhow::anyhow!( + "Can not perform a '--user' install. User site-packages are not visible in this virtualenv {}.", + venv.root().simplified_display().cyan() + )); + } + let _lock = venv.lock()?; // Determine the set of installed packages. From 642c98726d717e8bfacf8739eaa270df3b3b8d97 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 16:51:56 +0000 Subject: [PATCH 06/19] Update pip install `--user` arg docstring --- crates/uv/src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index ec51aa37b1c2..e7fd1a8874ff 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -797,12 +797,12 @@ struct PipInstallArgs { #[clap(long, requires = "discovery")] break_system_packages: bool, - /// Install packages into user site-packages directory + /// Install packages into user directory. /// - /// The `--user` option allows `uv` to install packages to a location that is specific to a - /// user. The user scheme can be customized by setting the `PYTHONUSERBASE` environment - /// variable. - #[clap(long, conflicts_with = "system")] + /// Install to the Python user install directory for your platform. + /// Typically ~/.local/, or %APPDATA%\Python on Windows. The install location can be customized by + /// setting the `PYTHONUSERBASE` environment variable. + #[clap(long, conflicts_with = "system", group = "discovery")] user: bool, /// Use legacy `setuptools` behavior when building source distributions without a From dcc7c29e63d6a8177a6d0cd5fbb1048368bd5b95 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 17:47:13 +0000 Subject: [PATCH 07/19] Support user site in pip list --- crates/uv-interpreter/src/python_environment.rs | 2 +- crates/uv/src/commands/pip_install.rs | 2 +- crates/uv/src/commands/pip_list.rs | 9 ++++++--- crates/uv/src/main.rs | 13 ++++++++++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index 5d79fcf389b2..ba32718f94f6 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -73,7 +73,7 @@ impl PythonEnvironment { } /// Create a [`PythonEnvironment`] with user scheme. - pub fn from_user_scheme(python: Option<&String>, platform: Platform, cache: &Cache) -> Result { + pub fn from_user_scheme(python: Option<&str>, platform: Platform, cache: &Cache) -> Result { // Attempt to determine the interpreter based on the provided criteria let interpreter = if let Some(requested_python) = python { // If a specific Python version is requested diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 7e13d9bdc440..3c4144f84de9 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -112,7 +112,7 @@ pub(crate) async fn pip_install( // Detect the current Python interpreter. let platform = Platform::current()?; let venv = if user { - PythonEnvironment::from_user_scheme(python.as_ref(), platform, &cache)? + PythonEnvironment::from_user_scheme(python.as_deref(), platform, &cache)? } else if let Some(python) = python.as_ref() { PythonEnvironment::from_requested_python(python, &platform, &cache)? } else if system { diff --git a/crates/uv/src/commands/pip_list.rs b/crates/uv/src/commands/pip_list.rs index dd329d66a5e2..d48d578cb0e4 100644 --- a/crates/uv/src/commands/pip_list.rs +++ b/crates/uv/src/commands/pip_list.rs @@ -31,15 +31,18 @@ pub(crate) fn pip_list( format: &ListFormat, python: Option<&str>, system: bool, + user: bool, cache: &Cache, printer: Printer, ) -> Result { // Detect the current Python interpreter. let platform = Platform::current()?; - let venv = if let Some(python) = python { - PythonEnvironment::from_requested_python(python, &platform, cache)? + let venv = if user { + PythonEnvironment::from_user_scheme(python, platform, &cache)? + } else if let Some(python) = python.as_ref() { + PythonEnvironment::from_requested_python(python, &platform, &cache)? } else if system { - PythonEnvironment::from_default_python(&platform, cache)? + PythonEnvironment::from_default_python(&platform, &cache)? } else { match PythonEnvironment::from_virtualenv(platform.clone(), cache) { Ok(venv) => venv, diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index e7fd1a8874ff..9e9a9ebff7cd 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -799,9 +799,11 @@ struct PipInstallArgs { /// Install packages into user directory. /// - /// Install to the Python user install directory for your platform. - /// Typically ~/.local/, or %APPDATA%\Python on Windows. The install location can be customized by - /// setting the `PYTHONUSERBASE` environment variable. + /// This option allows `uv` to install packages to user install directory for your platform. + /// The installation location can be customized by setting the `PYTHONUSERBASE` environment variable. + /// + /// Note: It is generally recommended to manage Python packages using a virtual environment + /// instead of installing them into the user or system Python directories. #[clap(long, conflicts_with = "system", group = "discovery")] user: bool, @@ -1034,6 +1036,10 @@ struct PipListArgs { /// should be used with caution. #[clap(long, conflicts_with = "python", group = "discovery")] system: bool, + + /// List packages installed in user site. + #[clap(long, conflicts_with = "system", group = "discovery")] + user: bool, } #[derive(Args)] @@ -1594,6 +1600,7 @@ async fn run() -> Result { &args.format, args.python.as_deref(), args.system, + args.user, &cache, printer, ), From 0bb5b6a8a826e3251ba26e0298c0ce6b5559e53c Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 18:02:08 +0000 Subject: [PATCH 08/19] Support user site for `pip show` --- crates/uv/src/commands/pip_show.rs | 5 ++++- crates/uv/src/main.rs | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/uv/src/commands/pip_show.rs b/crates/uv/src/commands/pip_show.rs index 2ed27248b26a..f2764df786ab 100644 --- a/crates/uv/src/commands/pip_show.rs +++ b/crates/uv/src/commands/pip_show.rs @@ -21,6 +21,7 @@ pub(crate) fn pip_show( strict: bool, python: Option<&str>, system: bool, + user: bool, cache: &Cache, printer: Printer, ) -> Result { @@ -39,7 +40,9 @@ pub(crate) fn pip_show( // Detect the current Python interpreter. let platform = Platform::current()?; - let venv = if let Some(python) = python { + let venv = if user { + PythonEnvironment::from_user_scheme(python, platform, cache)? + } else if let Some(python) = python.as_ref() { PythonEnvironment::from_requested_python(python, &platform, cache)? } else if system { PythonEnvironment::from_default_python(&platform, cache)? diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index 9e9a9ebff7cd..f04cd967f170 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -1073,7 +1073,7 @@ struct PipShowArgs { )] python: Option, - /// List packages for the system Python. + /// Show packages for the system Python. /// /// By default, `uv` lists packages in the currently activated virtual environment, or a virtual /// environment (`.venv`) located in the current working directory or any parent directory, @@ -1084,6 +1084,10 @@ struct PipShowArgs { /// should be used with caution. #[clap(long, conflicts_with = "python", group = "discovery")] system: bool, + + /// Show packages installed in user site. + #[clap(long, conflicts_with = "system", group = "discovery")] + user: bool, } #[derive(Args)] @@ -1611,6 +1615,7 @@ async fn run() -> Result { args.strict, args.python.as_deref(), args.system, + args.user, &cache, printer, ), From 7bc18957776890ca5a034efe3a840d7d86ea9583 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 18:05:52 +0000 Subject: [PATCH 09/19] Support user site for `pip freeze` --- crates/uv/src/commands/pip_freeze.rs | 5 ++++- crates/uv/src/main.rs | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/uv/src/commands/pip_freeze.rs b/crates/uv/src/commands/pip_freeze.rs index 8e3e97353f97..ff00aa55d170 100644 --- a/crates/uv/src/commands/pip_freeze.rs +++ b/crates/uv/src/commands/pip_freeze.rs @@ -20,12 +20,15 @@ pub(crate) fn pip_freeze( strict: bool, python: Option<&str>, system: bool, + user: bool, cache: &Cache, printer: Printer, ) -> Result { // Detect the current Python interpreter. let platform = Platform::current()?; - let venv = if let Some(python) = python { + let venv = if user { + PythonEnvironment::from_user_scheme(python, platform, cache)? + } else if let Some(python) = python { PythonEnvironment::from_requested_python(python, &platform, cache)? } else if system { PythonEnvironment::from_default_python(&platform, cache)? diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index f04cd967f170..49d358c722a0 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -979,6 +979,10 @@ struct PipFreezeArgs { /// should be used with caution. #[clap(long, conflicts_with = "python", group = "discovery")] system: bool, + + /// Use packages installed in user site. + #[clap(long, conflicts_with = "system", group = "discovery")] + user: bool, } #[derive(Args)] @@ -1591,6 +1595,7 @@ async fn run() -> Result { args.strict, args.python.as_deref(), args.system, + args.user, &cache, printer, ), From ad97a62b03ca64d57a5bac6d8d35091ed50185a9 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 18:06:26 +0000 Subject: [PATCH 10/19] Fix python arg type conversion --- crates/uv/src/commands/pip_list.rs | 2 +- crates/uv/src/commands/pip_show.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/uv/src/commands/pip_list.rs b/crates/uv/src/commands/pip_list.rs index d48d578cb0e4..17d8a5312459 100644 --- a/crates/uv/src/commands/pip_list.rs +++ b/crates/uv/src/commands/pip_list.rs @@ -39,7 +39,7 @@ pub(crate) fn pip_list( let platform = Platform::current()?; let venv = if user { PythonEnvironment::from_user_scheme(python, platform, &cache)? - } else if let Some(python) = python.as_ref() { + } else if let Some(python) = python { PythonEnvironment::from_requested_python(python, &platform, &cache)? } else if system { PythonEnvironment::from_default_python(&platform, &cache)? diff --git a/crates/uv/src/commands/pip_show.rs b/crates/uv/src/commands/pip_show.rs index f2764df786ab..3143df840f6d 100644 --- a/crates/uv/src/commands/pip_show.rs +++ b/crates/uv/src/commands/pip_show.rs @@ -42,7 +42,7 @@ pub(crate) fn pip_show( let platform = Platform::current()?; let venv = if user { PythonEnvironment::from_user_scheme(python, platform, cache)? - } else if let Some(python) = python.as_ref() { + } else if let Some(python) = python { PythonEnvironment::from_requested_python(python, &platform, cache)? } else if system { PythonEnvironment::from_default_python(&platform, cache)? From 947f93bb3b65f7175a8a234fbbceec870f265043 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 18:09:01 +0000 Subject: [PATCH 11/19] Format file --- crates/uv-interpreter/src/get_interpreter_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/uv-interpreter/src/get_interpreter_info.py b/crates/uv-interpreter/src/get_interpreter_info.py index 9719470248cb..7ed006f2e7a0 100644 --- a/crates/uv-interpreter/src/get_interpreter_info.py +++ b/crates/uv-interpreter/src/get_interpreter_info.py @@ -268,9 +268,9 @@ def _should_use_osx_framework_prefix() -> bool: or our own, and we deal with this special case in ``get_scheme()`` instead. """ return ( - "osx_framework_library" in _AVAILABLE_SCHEMES - and not running_under_virtualenv() - and is_osx_framework() + "osx_framework_library" in _AVAILABLE_SCHEMES + and not running_under_virtualenv() + and is_osx_framework() ) def _infer_user() -> str: From 2f905e467b0efcd253468a7093bf2b21ef146f75 Mon Sep 17 00:00:00 2001 From: Xin Date: Sun, 10 Mar 2024 23:08:31 +0000 Subject: [PATCH 12/19] Update comment for pip install user arg --- crates/uv/src/main.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index 49d358c722a0..60253949e497 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -799,11 +799,16 @@ struct PipInstallArgs { /// Install packages into user directory. /// - /// This option allows `uv` to install packages to user install directory for your platform. - /// The installation location can be customized by setting the `PYTHONUSERBASE` environment variable. + /// When enabled, this option instructs `uv` to install to install packages to user install + /// directory specific to the platform. The installation location can be overriden by the + /// `PYTHONUSERBASE` environment variable. /// - /// Note: It is generally recommended to manage Python packages using a virtual environment - /// instead of installing them into the user or system Python directories. + /// Searches for an interpreter in this order: `--python`/`-p` argument, active virtual + /// environment or the first Python in the system `PATH`. + /// + /// WARNING: `--user` is intended for use in continuous integration (CI) or multi-user machine. + /// It is generally recommended to manage Python packages using a virtual environment to + /// isolate pacakges. #[clap(long, conflicts_with = "system", group = "discovery")] user: bool, From 880da62896f58c05c6f468792115f5d0db4a46f9 Mon Sep 17 00:00:00 2001 From: Xin Date: Mon, 11 Mar 2024 07:22:11 +0000 Subject: [PATCH 13/19] Add comments to _infer_user --- crates/uv-interpreter/src/get_interpreter_info.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/uv-interpreter/src/get_interpreter_info.py b/crates/uv-interpreter/src/get_interpreter_info.py index 7ed006f2e7a0..dbe93617bff1 100644 --- a/crates/uv-interpreter/src/get_interpreter_info.py +++ b/crates/uv-interpreter/src/get_interpreter_info.py @@ -283,6 +283,8 @@ def _infer_user() -> str: suffixed = f"{os.name}_user" if suffixed in _AVAILABLE_SCHEMES: return suffixed + # Fall back to posix_user if user scheme is unavailable. + # `pip` would raise exeception when "posix_user" not in _AVAILABLE_SCHEMES return "posix_user" def _infer_prefix() -> str: From d2df185639cd5de1a4e7fd4cdb17d004a2f98fad Mon Sep 17 00:00:00 2001 From: Xin Date: Mon, 11 Mar 2024 07:38:18 +0000 Subject: [PATCH 14/19] Add include dir to ensure user scheme paths --- crates/uv-interpreter/src/python_environment.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index ba32718f94f6..4fa18bc786da 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -96,6 +96,7 @@ impl PythonEnvironment { interpreter_with_user_scheme.purelib(), interpreter_with_user_scheme.scripts(), interpreter_with_user_scheme.data(), + interpreter_with_user_scheme.include(), ]; for path in directories { From eb388901a6681765577c42b74f6877dac9eda1dc Mon Sep 17 00:00:00 2001 From: Xin Date: Mon, 11 Mar 2024 08:29:49 +0000 Subject: [PATCH 15/19] Run cargo fmt --all --- crates/uv-interpreter/src/cfg.rs | 10 ++++++++-- crates/uv-interpreter/src/python_environment.rs | 6 +++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/uv-interpreter/src/cfg.rs b/crates/uv-interpreter/src/cfg.rs index 50b802f6e92c..d7f741d84488 100644 --- a/crates/uv-interpreter/src/cfg.rs +++ b/crates/uv-interpreter/src/cfg.rs @@ -43,7 +43,11 @@ impl PyVenvConfiguration { } } - Ok(Self { virtualenv, uv, include_system_site_packages }) + Ok(Self { + virtualenv, + uv, + include_system_site_packages, + }) } /// Returns true if the virtual environment was created with the `virtualenv` package. @@ -57,7 +61,9 @@ impl PyVenvConfiguration { } /// Return true if the virtual environment has access to system site packages. - pub fn include_system_site_packages(&self) -> bool { self.include_system_site_packages } + pub fn include_system_site_packages(&self) -> bool { + self.include_system_site_packages + } } #[derive(Debug, Error)] diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index 4fa18bc786da..e4783f429e12 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -73,7 +73,11 @@ impl PythonEnvironment { } /// Create a [`PythonEnvironment`] with user scheme. - pub fn from_user_scheme(python: Option<&str>, platform: Platform, cache: &Cache) -> Result { + pub fn from_user_scheme( + python: Option<&str>, + platform: Platform, + cache: &Cache, + ) -> Result { // Attempt to determine the interpreter based on the provided criteria let interpreter = if let Some(requested_python) = python { // If a specific Python version is requested From 97bf3368e95c560e3cfbf0233a2c0888a1be1a81 Mon Sep 17 00:00:00 2001 From: Xin Date: Mon, 11 Mar 2024 08:37:15 +0000 Subject: [PATCH 16/19] Fix CI errors --- crates/uv-interpreter/src/cfg.rs | 2 +- crates/uv/src/commands/pip_list.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/uv-interpreter/src/cfg.rs b/crates/uv-interpreter/src/cfg.rs index d7f741d84488..0660d9442965 100644 --- a/crates/uv-interpreter/src/cfg.rs +++ b/crates/uv-interpreter/src/cfg.rs @@ -37,7 +37,7 @@ impl PyVenvConfiguration { uv = true; } "include-system-site-packages" => { - include_system_site_packages = value.trim() == "true" + include_system_site_packages = value.trim() == "true"; } _ => {} } diff --git a/crates/uv/src/commands/pip_list.rs b/crates/uv/src/commands/pip_list.rs index 17d8a5312459..f205c27354dd 100644 --- a/crates/uv/src/commands/pip_list.rs +++ b/crates/uv/src/commands/pip_list.rs @@ -38,11 +38,11 @@ pub(crate) fn pip_list( // Detect the current Python interpreter. let platform = Platform::current()?; let venv = if user { - PythonEnvironment::from_user_scheme(python, platform, &cache)? + PythonEnvironment::from_user_scheme(python, platform, cache)? } else if let Some(python) = python { - PythonEnvironment::from_requested_python(python, &platform, &cache)? + PythonEnvironment::from_requested_python(python, &platform, cache)? } else if system { - PythonEnvironment::from_default_python(&platform, &cache)? + PythonEnvironment::from_default_python(&platform, cache)? } else { match PythonEnvironment::from_virtualenv(platform.clone(), cache) { Ok(venv) => venv, From 1991f70197d00db22eb7fa25ba3ad5f4f781a100 Mon Sep 17 00:00:00 2001 From: Xin Date: Mon, 11 Mar 2024 09:03:33 +0000 Subject: [PATCH 17/19] Update crates/uv-interpreter/src/python_environment.rs Co-authored-by: konsti --- crates/uv-interpreter/src/python_environment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index e4783f429e12..03ab0b9ab7a0 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -105,7 +105,7 @@ impl PythonEnvironment { for path in directories { if !Path::new(path).exists() { - fs::create_dir_all(path)?; + fs_err::create_dir_all(path)?; } } From 6749fa78c04b6cb6433ceb9f5dc1c7cf347e2c32 Mon Sep 17 00:00:00 2001 From: Xin Date: Mon, 11 Mar 2024 11:14:41 +0000 Subject: [PATCH 18/19] Use fs_err instead of fs --- crates/uv-interpreter/src/python_environment.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index 03ab0b9ab7a0..5b6dca25d46a 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -1,5 +1,6 @@ +use fs_err as fs; +use std::env; use std::path::{Path, PathBuf}; -use std::{env, fs}; use tracing::debug; @@ -105,7 +106,7 @@ impl PythonEnvironment { for path in directories { if !Path::new(path).exists() { - fs_err::create_dir_all(path)?; + fs::create_dir_all(path)?; } } From 5f49ffcdf2e15504896d3db751c79005e5e5df67 Mon Sep 17 00:00:00 2001 From: Xin Date: Wed, 13 Mar 2024 23:57:30 +0000 Subject: [PATCH 19/19] Fix merge issues --- crates/uv-interpreter/src/interpreter.rs | 118 ++++++------------ .../uv-interpreter/src/python_environment.rs | 14 +-- crates/uv/src/commands/pip_list.rs | 5 +- crates/uv/src/commands/pip_show.rs | 5 +- 4 files changed, 45 insertions(+), 97 deletions(-) diff --git a/crates/uv-interpreter/src/interpreter.rs b/crates/uv-interpreter/src/interpreter.rs index 6e726454bf29..e519be8366e6 100644 --- a/crates/uv-interpreter/src/interpreter.rs +++ b/crates/uv-interpreter/src/interpreter.rs @@ -1,5 +1,5 @@ use std::path::{Path, PathBuf}; -use std::process::{Command, Output}; +use std::process::Command; use configparser::ini::Ini; use fs_err as fs; @@ -103,8 +103,8 @@ impl Interpreter { } /// Return a new [`Interpreter`] with user scheme. - pub fn with_user_scheme(self) -> Result { - let info = InterpreterInfo::query_user_scheme_info(self.sys_executable())?; + pub fn with_user_scheme(self, cache: &Cache) -> Result { + let info = InterpreterInfo::query_user_scheme_info(self.sys_executable(), cache)?; Ok(Self { scheme: info.scheme, ..self @@ -534,90 +534,25 @@ impl InterpreterInfo { } /// Return the [`InterpreterInfo`] for the given Python interpreter with user scheme. - pub(crate) fn query_user_scheme_info(interpreter: &Path) -> Result { - let script = include_str!("get_interpreter_info.py"); + pub(crate) fn query_user_scheme_info(interpreter: &Path, cache: &Cache) -> Result { let envs = vec![("_UV_USE_USER_SCHEME", "1")]; - let output = Self::execute_script(interpreter, script, Some(envs))?; - - let data: Self = serde_json::from_slice(&output.stdout).map_err(|err| { - Error::PythonSubcommandOutput { - message: format!( - "Querying Python at `{}` did not return the expected data: {err}", - interpreter.display(), - ), - exit_code: output.status, - stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), - stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), - } - })?; - - Ok(data) - } - - /// Execute the given Python script using an interpreter. - fn execute_script( - interpreter: &Path, - script: &str, - envs: Option>, - ) -> Result { - let mut command = Command::new(interpreter); - - if let Some(env_variables) = envs { - command.envs(env_variables); - } - - let script_clone = script.to_string(); - let output = if cfg!(windows) - && interpreter - .extension() - .is_some_and(|extension| extension == "bat") - { - // Multiline arguments aren't well-supported in batch files and `pyenv-win`, for example, trips over it. - // We work around this batch limitation by passing the script via stdin instead. - // This is somewhat more expensive because we have to spawn a new thread to write the - // stdin to avoid deadlocks in case the child process waits for the parent to read stdout. - // The performance overhead is the reason why we only applies this to batch files. - // https://github.com/pyenv-win/pyenv-win/issues/589 - let mut child = command - .arg("-") - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .spawn() - .map_err(|err| Error::PythonSubcommandLaunch { - interpreter: interpreter.to_path_buf(), - err, - })?; - - let mut stdin = child.stdin.take().unwrap(); - - // From the Rust documentation: - // If the child process fills its stdout buffer, it may end up - // waiting until the parent reads the stdout, and not be able to - // read stdin in the meantime, causing a deadlock. - // Writing from another thread ensures that stdout is being read - // at the same time, avoiding the problem. - std::thread::spawn(move || { - stdin - .write_all(script_clone.as_bytes()) - .expect("failed to write to stdin"); - }); + let tempdir = tempfile::tempdir_in(cache.root())?; + Self::setup_python_query_files(tempdir.path())?; - child.wait_with_output() - } else { - command.arg("-c").arg(script_clone).output() - } - .map_err(|err| Error::PythonSubcommandLaunch { - interpreter: interpreter.to_path_buf(), - err, - })?; + let output = Command::new(interpreter) + .arg("-m") + .arg("python.get_interpreter_info") + .envs(envs) + .current_dir(tempdir.path().simplified()) + .output() + .map_err(|err| Error::PythonSubcommandLaunch { + interpreter: interpreter.to_path_buf(), + err, + })?; // stderr isn't technically a criterion for success, but i don't know of any cases where there // should be stderr output and if there is, we want to know if !output.status.success() || !output.stderr.is_empty() { - if output.status.code() == Some(3) { - return Err(Error::Python2OrOlder); - } - return Err(Error::PythonSubcommandOutput { message: format!( "Querying Python at `{}` failed with status {}", @@ -630,7 +565,26 @@ impl InterpreterInfo { }); } - Ok(output) + let result: InterpreterInfoResult = + serde_json::from_slice(&output.stdout).map_err(|err| { + Error::PythonSubcommandOutput { + message: format!( + "Querying Python at `{}` did not return the expected data: {err}", + interpreter.display(), + ), + exit_code: output.status, + stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), + stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), + } + })?; + + match result { + InterpreterInfoResult::Error(err) => Err(Error::QueryScript { + err, + interpreter: interpreter.to_path_buf(), + }), + InterpreterInfoResult::Success(data) => Ok(*data), + } } } diff --git a/crates/uv-interpreter/src/python_environment.rs b/crates/uv-interpreter/src/python_environment.rs index 4645f2db7cb1..b3c0a55e3fa7 100644 --- a/crates/uv-interpreter/src/python_environment.rs +++ b/crates/uv-interpreter/src/python_environment.rs @@ -69,25 +69,21 @@ impl PythonEnvironment { } /// Create a [`PythonEnvironment`] with user scheme. - pub fn from_user_scheme( - python: Option<&str>, - platform: Platform, - cache: &Cache, - ) -> Result { + pub fn from_user_scheme(python: Option<&str>, cache: &Cache) -> Result { // Attempt to determine the interpreter based on the provided criteria let interpreter = if let Some(requested_python) = python { // If a specific Python version is requested - Self::from_requested_python(requested_python, &platform, cache)?.interpreter + Self::from_requested_python(requested_python, cache)?.interpreter } else if let Some(_venv) = detect_virtual_env()? { // If a virtual environment is detected - Self::from_virtualenv(platform, cache)?.interpreter + Self::from_virtualenv(cache)?.interpreter } else { // Fallback to the default Python interpreter - Self::from_default_python(&platform, cache)?.interpreter + Self::from_default_python(cache)?.interpreter }; // Apply the user scheme to the determined interpreter - let interpreter_with_user_scheme = interpreter.with_user_scheme()?; + let interpreter_with_user_scheme = interpreter.with_user_scheme(cache)?; // Ensure interpreter scheme directories exist, as per the model in pip // diff --git a/crates/uv/src/commands/pip_list.rs b/crates/uv/src/commands/pip_list.rs index 5d2fb615e9e5..7bd1f10e4cec 100644 --- a/crates/uv/src/commands/pip_list.rs +++ b/crates/uv/src/commands/pip_list.rs @@ -35,11 +35,10 @@ pub(crate) fn pip_list( printer: Printer, ) -> Result { // Detect the current Python interpreter. - let platform = Platform::current()?; let venv = if user { - PythonEnvironment::from_user_scheme(python, platform, cache)? + PythonEnvironment::from_user_scheme(python, cache)? } else if let Some(python) = python { - PythonEnvironment::from_requested_python(python, &platform, cache)? + PythonEnvironment::from_requested_python(python, cache)? } else if system { PythonEnvironment::from_default_python(cache)? } else { diff --git a/crates/uv/src/commands/pip_show.rs b/crates/uv/src/commands/pip_show.rs index c5d244c14879..58d291ab70b3 100644 --- a/crates/uv/src/commands/pip_show.rs +++ b/crates/uv/src/commands/pip_show.rs @@ -40,11 +40,10 @@ pub(crate) fn pip_show( } // Detect the current Python interpreter. - let platform = Platform::current()?; let venv = if user { - PythonEnvironment::from_user_scheme(python, platform, cache)? + PythonEnvironment::from_user_scheme(python, cache)? } else if let Some(python) = python { - PythonEnvironment::from_requested_python(python, &platform, cache)? + PythonEnvironment::from_requested_python(python, cache)? } else if system { PythonEnvironment::from_default_python(cache)? } else {