From a39eb61ade85445095658075c0b4cabb16b5a41b Mon Sep 17 00:00:00 2001 From: konsti Date: Thu, 29 Aug 2024 22:48:22 +0200 Subject: [PATCH] Use windows registry to discover python (#6761) Our current strategy of parsing the output of `py --list-paths` to get the installed python versions on windows is brittle (#6524, missing `py`, etc.) and it's slow (10ms last time i measured). Instead, we should behave spec-compliant and read the python versions from the registry following PEP 514. It's not fully clear which errors we should ignore and which ones we need to raise. We're using the official rust-for-windows crates for accessing the registry. Fixes #1521 Fixes #6524 --- Cargo.lock | 2 + Cargo.toml | 2 + crates/uv-python/Cargo.toml | 2 + crates/uv-python/src/discovery.rs | 75 +++++++----- crates/uv-python/src/lib.rs | 4 +- crates/uv-python/src/py_launcher.rs | 171 +++++++++++++++------------- crates/uv-settings/src/settings.rs | 4 +- docs/concepts/python-versions.md | 4 +- docs/reference/settings.md | 4 +- uv.schema.json | 2 +- 10 files changed, 151 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc30ef3f415c..beecd23131cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5043,6 +5043,8 @@ dependencies = [ "uv-state", "uv-warnings", "which", + "windows-registry", + "windows-result", "windows-sys 0.59.0", "winsafe 0.0.22", ] diff --git a/Cargo.toml b/Cargo.toml index 155da3443de1..7557cc46c0b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,6 +157,8 @@ url = { version = "2.5.0" } urlencoding = { version = "2.1.3" } walkdir = { version = "2.5.0" } which = { version = "6.0.0", features = ["regex"] } +windows-registry = { version = "0.2.0" } +windows-result = { version = "0.2.0" } windows-sys = { version = "0.59.0", features = ["Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem", "Win32_System_Ioctl", "Win32_System_IO"] } winreg = { version = "0.52.0" } winsafe = { version = "0.0.22", features = ["kernel"] } diff --git a/crates/uv-python/Cargo.toml b/crates/uv-python/Cargo.toml index 57bcd183feae..c3c17a6ce952 100644 --- a/crates/uv-python/Cargo.toml +++ b/crates/uv-python/Cargo.toml @@ -58,6 +58,8 @@ rustix = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] windows-sys = { workspace = true } winsafe = { workspace = true } +windows-registry = { workspace = true } +windows-result = { workspace = true } [dev-dependencies] anyhow = { version = "1.0.80" } diff --git a/crates/uv-python/src/discovery.rs b/crates/uv-python/src/discovery.rs index 9158d9837ab2..0e36010db15b 100644 --- a/crates/uv-python/src/discovery.rs +++ b/crates/uv-python/src/discovery.rs @@ -20,7 +20,8 @@ use crate::implementation::ImplementationName; use crate::installation::PythonInstallation; use crate::interpreter::Error as InterpreterError; use crate::managed::ManagedPythonInstallations; -use crate::py_launcher::{self, py_list_paths}; +#[cfg(windows)] +use crate::py_launcher::registry_pythons; use crate::virtualenv::{ conda_prefix_from_env, virtualenv_from_env, virtualenv_from_working_dir, virtualenv_python_executable, @@ -164,8 +165,8 @@ pub enum PythonSource { DiscoveredEnvironment, /// An executable was found in the search path i.e. `PATH` SearchPath, - /// An executable was found via the `py` launcher - PyLauncher, + /// An executable was found in the Windows registry via PEP 514 + Registry, /// The Python installation was found in the uv managed Python directory Managed, /// The Python installation was found via the invoking interpreter i.e. via `python -m uv ...` @@ -189,9 +190,9 @@ pub enum Error { #[error(transparent)] VirtualEnv(#[from] crate::virtualenv::Error), - /// An error was encountered when using the `py` launcher on Windows. - #[error(transparent)] - PyLauncher(#[from] crate::py_launcher::Error), + #[cfg(windows)] + #[error("Failed to query installed Python versions from the Windows registry")] + RegistryError(#[from] windows_result::Error), /// An invalid version request was given #[error("Invalid version request: {0}")] @@ -307,23 +308,40 @@ fn python_executables_from_installed<'a>( }) .flatten(); - // TODO(konstin): Implement to read python installations from the registry instead. let from_py_launcher = std::iter::once_with(move || { - (cfg!(windows) && env::var_os("UV_TEST_PYTHON_PATH").is_none()) - .then(|| { - py_list_paths() - .map(|entries| - // We can avoid querying the interpreter using versions from the py launcher output unless a patch is requested - entries.into_iter().filter(move |entry| - version.is_none() || version.is_some_and(|version| - version.has_patch() || version.matches_major_minor(entry.major, entry.minor) - ) - ) - .map(|entry| (PythonSource::PyLauncher, entry.executable_path))) - .map_err(Error::from) - }) - .into_iter() - .flatten_ok() + #[cfg(windows)] + { + env::var_os("UV_TEST_PYTHON_PATH") + .is_none() + .then(|| { + registry_pythons() + .map(|entries| { + entries + .into_iter() + .filter(move |entry| { + // Skip interpreter probing if we already know the version + // doesn't match. + if let Some(version_request) = version { + if let Some(version) = &entry.version { + version_request.matches_version(version) + } else { + true + } + } else { + true + } + }) + .map(|entry| (PythonSource::Registry, entry.path)) + }) + .map_err(Error::from) + }) + .into_iter() + .flatten_ok() + } + #[cfg(not(windows))] + { + Vec::new() + } }) .flatten(); @@ -626,11 +644,6 @@ impl Error { false } }, - // Ignore `py` if it's not installed - Error::PyLauncher(py_launcher::Error::NotFound) => { - debug!("The `py` launcher could not be found to query for Python versions"); - false - } _ => true, } } @@ -1293,7 +1306,7 @@ impl PythonPreference { // If not dealing with a system interpreter source, we don't care about the preference if !matches!( source, - PythonSource::Managed | PythonSource::SearchPath | PythonSource::PyLauncher + PythonSource::Managed | PythonSource::SearchPath | PythonSource::Registry ) { return true; } @@ -1302,10 +1315,10 @@ impl PythonPreference { PythonPreference::OnlyManaged => matches!(source, PythonSource::Managed), Self::Managed | Self::System => matches!( source, - PythonSource::Managed | PythonSource::SearchPath | PythonSource::PyLauncher + PythonSource::Managed | PythonSource::SearchPath | PythonSource::Registry ), PythonPreference::OnlySystem => { - matches!(source, PythonSource::SearchPath | PythonSource::PyLauncher) + matches!(source, PythonSource::SearchPath | PythonSource::Registry) } } } @@ -1619,7 +1632,7 @@ impl fmt::Display for PythonSource { Self::CondaPrefix => f.write_str("conda prefix"), Self::DiscoveredEnvironment => f.write_str("virtual environment"), Self::SearchPath => f.write_str("search path"), - Self::PyLauncher => f.write_str("`py` launcher output"), + Self::Registry => f.write_str("registry"), Self::Managed => f.write_str("managed installations"), Self::ParentInterpreter => f.write_str("parent interpreter"), } diff --git a/crates/uv-python/src/lib.rs b/crates/uv-python/src/lib.rs index 2ba1e90f2791..a6730e474aae 100644 --- a/crates/uv-python/src/lib.rs +++ b/crates/uv-python/src/lib.rs @@ -29,6 +29,7 @@ pub mod managed; pub mod platform; mod pointer_size; mod prefix; +#[cfg(windows)] mod py_launcher; mod python_version; mod target; @@ -60,9 +61,6 @@ pub enum Error { #[error(transparent)] Discovery(#[from] discovery::Error), - #[error(transparent)] - PyLauncher(#[from] py_launcher::Error), - #[error(transparent)] ManagedPython(#[from] managed::Error), diff --git a/crates/uv-python/src/py_launcher.rs b/crates/uv-python/src/py_launcher.rs index e7cc4a7b7625..ae9fd09eb1f1 100644 --- a/crates/uv-python/src/py_launcher.rs +++ b/crates/uv-python/src/py_launcher.rs @@ -1,93 +1,108 @@ -use regex::Regex; -use std::io; +use crate::PythonVersion; use std::path::PathBuf; -use std::process::{Command, ExitStatus}; -use std::sync::LazyLock; -use thiserror::Error; -use tracing::info_span; +use std::str::FromStr; +use tracing::debug; +use windows_registry::{Key, Value, CURRENT_USER, LOCAL_MACHINE}; +/// A Python interpreter found in the Windows registry through PEP 514. +/// +/// There are a lot more (optional) fields defined in PEP 514, but we only care about path and +/// version here, for everything else we probe with a Python script. #[derive(Debug, Clone)] -pub(crate) struct PyListPath { - pub(crate) major: u8, - pub(crate) minor: u8, - pub(crate) executable_path: PathBuf, +pub(crate) struct RegistryPython { + pub(crate) path: PathBuf, + pub(crate) version: Option, } -/// An error was encountered when using the `py` launcher on Windows. -#[derive(Error, Debug)] -pub enum Error { - #[error("{message} with {exit_code}\n--- stdout:\n{stdout}\n--- stderr:\n{stderr}\n---")] - StatusCode { - message: String, - exit_code: ExitStatus, - stdout: String, - stderr: String, - }, - #[error("Failed to run `py --list-paths` to find Python installations")] - Io(#[source] io::Error), - #[error("The `py` launcher could not be found")] - NotFound, +/// Àdding `windows_registry::Value::into_string()`. +fn value_to_string(value: Value) -> Option { + match value { + Value::String(string) => Some(string), + Value::Bytes(bytes) => String::from_utf8(bytes.clone()).ok(), + Value::U32(_) | Value::U64(_) | Value::MultiString(_) | Value::Unknown(_) => None, + } } -/// ```text -/// -V:3.12 C:\Users\Ferris\AppData\Local\Programs\Python\Python312\python.exe -/// -V:3.8 C:\Users\Ferris\AppData\Local\Programs\Python\Python38\python.exe -/// ``` -static PY_LIST_PATHS: LazyLock = LazyLock::new(|| { - // Without the `R` flag, paths have trailing \r - Regex::new(r"(?mR)^ -(?:V:)?(\d).(\d+)-?(?:arm)?\d*\s*\*?\s*(.*)$").unwrap() -}); - -/// Use the `py` launcher to find installed Python versions. -/// -/// Calls `py --list-paths`. -pub(crate) fn py_list_paths() -> Result, Error> { - // konstin: The command takes 8ms on my machine. - let output = info_span!("py_list_paths") - .in_scope(|| Command::new("py").arg("--list-paths").output()) - .map_err(|err| { - if err.kind() == std::io::ErrorKind::NotFound { - Error::NotFound - } else { - Error::Io(err) +/// Find all Pythons registered in the Windows registry following PEP 514. +pub(crate) fn registry_pythons() -> Result, windows_result::Error> { + let mut registry_pythons = Vec::new(); + for root_key in [CURRENT_USER, LOCAL_MACHINE] { + let Ok(key_python) = root_key.open(r"Software\Python") else { + continue; + }; + for company in key_python.keys()? { + // Reserved name according to the PEP. + if company == "PyLauncher" { + continue; } - })?; + let Ok(company_key) = key_python.open(&company) else { + // Ignore invalid entries + continue; + }; + for tag in company_key.keys()? { + let tag_key = company_key.open(&tag)?; - // `py` sometimes prints "Installed Pythons found by py Launcher for Windows" to stderr which we ignore. - if !output.status.success() { - return Err(Error::StatusCode { - message: format!( - "Running `py --list-paths` failed with status {}", - output.status - ), - exit_code: output.status, - stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(), - stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), - }); + if let Some(registry_python) = read_registry_entry(&company, &tag, &tag_key) { + registry_pythons.push(registry_python); + } + } + } } - // Find the first python of the version we want in the list - let stdout = String::from_utf8(output.stdout).map_err(|err| Error::StatusCode { - message: format!("The stdout of `py --list-paths` isn't UTF-8 encoded: {err}"), - exit_code: output.status, - stdout: String::from_utf8_lossy(err.as_bytes()).trim().to_string(), - stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(), - })?; + // The registry has no natural ordering, so we're processing the latest version first. + registry_pythons.sort_by(|a, b| { + // Highest version first (reverse), but entries without version at the bottom (regular + // order). + if let (Some(version_a), Some(version_b)) = (&a.version, &b.version) { + version_a.cmp(version_b).reverse().then(a.path.cmp(&b.path)) + } else { + a.version + .as_ref() + .map(|version| &***version) + .cmp(&b.version.as_ref().map(|version| &***version)) + .then(a.path.cmp(&b.path)) + } + }); - Ok(PY_LIST_PATHS - .captures_iter(&stdout) - .filter_map(|captures| { - let (_, [major, minor, path]) = captures.extract(); - if let (Some(major), Some(minor)) = (major.parse::().ok(), minor.parse::().ok()) - { - Some(PyListPath { - major, - minor, - executable_path: PathBuf::from(path), - }) - } else { + Ok(registry_pythons) +} + +fn read_registry_entry(company: &str, tag: &str, tag_key: &Key) -> Option { + // `ExecutablePath` is mandatory for executable Pythons. + let Some(executable_path) = tag_key + .open("InstallPath") + .and_then(|install_path| install_path.get_value("ExecutablePath")) + .ok() + .and_then(value_to_string) + else { + debug!( + r"Python interpreter in the registry is not executable: `Software\Python\{}\{}", + company, tag + ); + return None; + }; + + // `SysVersion` is optional. + let version = tag_key + .get_value("SysVersion") + .ok() + .and_then(|value| match value { + Value::String(s) => Some(s), + _ => None, + }) + .and_then(|s| match PythonVersion::from_str(&s) { + Ok(version) => Some(version), + Err(err) => { + debug!( + "Skipping Python interpreter ({executable_path}) \ + with invalid registry version {s}: {err}", + ); None } - }) - .collect()) + }); + + Some(RegistryPython { + path: PathBuf::from(executable_path), + version, + }) } diff --git a/crates/uv-settings/src/settings.rs b/crates/uv-settings/src/settings.rs index 53ae970ae41e..a4ead9181e80 100644 --- a/crates/uv-settings/src/settings.rs +++ b/crates/uv-settings/src/settings.rs @@ -591,8 +591,8 @@ pub struct PipOptions { /// workflows. /// /// Supported formats: - /// - `3.10` looks for an installed Python 3.10 using `py --list-paths` on Windows, or - /// `python3.10` on Linux and macOS. + /// - `3.10` looks for an installed Python 3.10 in the registry on Windows (see + /// `py --list-paths`), or `python3.10` on Linux and macOS. /// - `python3.10` or `python.exe` looks for a binary with the given name in `PATH`. /// - `/home/ferris/.local/bin/python3.10` uses the exact Python at the given path. #[option( diff --git a/docs/concepts/python-versions.md b/docs/concepts/python-versions.md index 8ff9cbfe1ea1..8553795ba6d8 100644 --- a/docs/concepts/python-versions.md +++ b/docs/concepts/python-versions.md @@ -169,8 +169,8 @@ When searching for a Python version, the following locations are checked: - Managed Python installations in the `UV_PYTHON_INSTALL_DIR`. - A Python interpreter on the `PATH` as `python`, `python3`, or `python3.x` on macOS and Linux, or `python.exe` on Windows. -- On Windows, the Python interpreter returned by `py --list-paths` that matches the requested - version. +- On Windows, the Python interpreters in the Windows registry and Microsoft Store Python + interpreters (see `py --list-paths`) that match the requested version. In some cases, uv allows using a Python version from a virtual environment. In this case, the virtual environment's interpreter will be checked for compatibility with the request before diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 1f6dd3f668be..e35330848348 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -2328,8 +2328,8 @@ which is intended for use in continuous integration (CI) environments or other a workflows. Supported formats: -- `3.10` looks for an installed Python 3.10 using `py --list-paths` on Windows, or - `python3.10` on Linux and macOS. +- `3.10` looks for an installed Python 3.10 in the registry on Windows (see + `py --list-paths`), or `python3.10` on Linux and macOS. - `python3.10` or `python.exe` looks for a binary with the given name in `PATH`. - `/home/ferris/.local/bin/python3.10` uses the exact Python at the given path. diff --git a/uv.schema.json b/uv.schema.json index c9efb1b5e53f..d983c13e44e3 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -887,7 +887,7 @@ ] }, "python": { - "description": "The Python interpreter into which packages should be installed.\n\nBy default, uv installs into the virtual environment in the current working directory or any parent directory. The `--python` option allows you to specify a different interpreter, which is intended for use in continuous integration (CI) environments or other automated workflows.\n\nSupported formats: - `3.10` looks for an installed Python 3.10 using `py --list-paths` on Windows, or `python3.10` on Linux and macOS. - `python3.10` or `python.exe` looks for a binary with the given name in `PATH`. - `/home/ferris/.local/bin/python3.10` uses the exact Python at the given path.", + "description": "The Python interpreter into which packages should be installed.\n\nBy default, uv installs into the virtual environment in the current working directory or any parent directory. The `--python` option allows you to specify a different interpreter, which is intended for use in continuous integration (CI) environments or other automated workflows.\n\nSupported formats: - `3.10` looks for an installed Python 3.10 in the registry on Windows (see `py --list-paths`), or `python3.10` on Linux and macOS. - `python3.10` or `python.exe` looks for a binary with the given name in `PATH`. - `/home/ferris/.local/bin/python3.10` uses the exact Python at the given path.", "type": [ "string", "null"