From bd9610140b9f4183ebb48c8de8c36a50549b086f Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 11 May 2023 16:45:33 +0800 Subject: [PATCH 1/4] Credential-related config options should utilize keyring if it is installed Fixes #1908 Signed-off-by: Frost Ming --- docs/docs/usage/config.md | 25 ++++++++++++++ news/1908.feature.md | 1 + src/pdm/_types.py | 31 ++++++++++++++--- src/pdm/cli/commands/publish/repository.py | 13 ++++---- src/pdm/models/auth.py | 34 +++++++++++++++++++ src/pdm/project/config.py | 6 ++++ tests/cli/conftest.py | 27 +++++++++++++++ tests/cli/test_config.py | 39 ++++++++++++++++++++++ tests/cli/test_publish.py | 6 ++++ 9 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 news/1908.feature.md diff --git a/docs/docs/usage/config.md b/docs/docs/usage/config.md index 06e7c52f0e..726c93c1b1 100644 --- a/docs/docs/usage/config.md +++ b/docs/docs/usage/config.md @@ -197,6 +197,15 @@ password = "" ca_certs = "/path/to/custom-cacerts.pem" ``` +Alternatively, these credentials can be provided with env vars: + +```bash +export PDM_PUBLISH_REPO=... +export PDM_PUBLISH_USERNAME=... +export PDM_PUBLISH_PASSWORD=... +export PDM_PUBLISH_CA_CERTS=... +``` + A PEM-encoded Certificate Authority bundle (`ca_certs`) can be used for local / custom PyPI repositories where the server certificate is not signed by the standard [certifi](https://github.com/certifi/python-certifi/blob/master/certifi/cacert.pem) CA bundle. !!! NOTE @@ -217,6 +226,22 @@ pdm config repository.company.url "https://pypi.company.org/legacy/" pdm config repository.company.ca_certs "/path/to/custom-cacerts.pem" ``` +## Password management with keyring + +When keyring is available and supported, the passwords will be stored to and retrieved from the keyring instead of writing to the config file. This supports both indexes and upload repositories. The service name will be `pdm-pypi-` for an index and `pdm-repository-` for a repository. + +You can enable keyring by either installing `keyring` into the same environment as PDM or installing globally. To add keyring to the PDM environment: + +```bash +pdm self add keyring +``` + +Alternatively, if you have installed a copy of keyring globally, make sure the CLI is exposed in the `PATH` env var to make it discoverable by PDM: + +```bash +export PATH=$PATH:path/to/keyring/bin +``` + ## Override the resolved package versions _New in version 1.12.0_ diff --git a/news/1908.feature.md b/news/1908.feature.md new file mode 100644 index 0000000000..38d03e712e --- /dev/null +++ b/news/1908.feature.md @@ -0,0 +1 @@ +When keyring is available, either by importing or by CLI, the credentials of repositories and PyPI indexes will be saved into it. diff --git a/src/pdm/_types.py b/src/pdm/_types.py index c65ab0596e..df63ec5124 100644 --- a/src/pdm/_types.py +++ b/src/pdm/_types.py @@ -1,24 +1,47 @@ from __future__ import annotations -import dataclasses +import dataclasses as dc from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Tuple, TypeVar, Union if TYPE_CHECKING: from typing import Protocol -@dataclasses.dataclass -class RepositoryConfig: +@dc.dataclass +class _RepositoryConfig: + """Private dataclass to be subclassed""" + config_prefix: str name: str url: str | None = None username: str | None = None - password: str | None = None + _password: str | None = dc.field(default=None, repr=False) verify_ssl: bool | None = None type: str | None = None ca_certs: str | None = None + +class RepositoryConfig(_RepositoryConfig): + def __init__(self, *args: Any, password: str | None = None, **kwargs: Any) -> None: + kwargs["_password"] = password + super().__init__(*args, **kwargs) + + @property + def password(self) -> str | None: + if self._password is None: + from pdm.models.auth import keyring + + service = f"pdm-{self.config_prefix}-{self.name}" + result = keyring.get_auth_info(service, self.username) + if result is not None: + self._password = result[1] + return self._password + + @password.setter + def password(self, value: str) -> None: + self._password = value + def passive_update(self, other: RepositoryConfig | None = None, **kwargs: Any) -> None: """An update method that prefers the existing value over the new one.""" if other is not None: diff --git a/src/pdm/cli/commands/publish/repository.py b/src/pdm/cli/commands/publish/repository.py index 55b5d1faef..84bcbfd7e8 100644 --- a/src/pdm/cli/commands/publish/repository.py +++ b/src/pdm/cli/commands/publish/repository.py @@ -9,16 +9,14 @@ import requests import requests_toolbelt import rich.progress -from unearth.auth import get_keyring_provider from pdm import termui from pdm.cli.commands.publish.package import PackageFile from pdm.exceptions import PdmUsageError +from pdm.models.auth import keyring from pdm.project import Project from pdm.project.config import DEFAULT_REPOSITORIES -keyring = get_keyring_provider() - class Repository: def __init__( @@ -45,13 +43,17 @@ def _ensure_credentials(self, username: str | None, password: str | None) -> tup return username, password if password: return "__token__", password + if keyring.enabled: + auth = keyring.get_auth_info(self.url, username) + if auth is not None: + return auth token = self._get_pypi_token_via_oidc() if token is not None: return "__token__", token if not termui.is_interactive(): raise PdmUsageError("Username and password are required") username, password, save = self._prompt_for_credentials(netloc, username) - if save and keyring is not None and termui.confirm("Save credentials to keyring?"): + if save and keyring.enabled and termui.confirm("Save credentials to keyring?"): self._credentials_to_save = (netloc, username, password) return username, password @@ -89,7 +91,7 @@ def _get_pypi_token_via_oidc(self) -> str | None: return token def _prompt_for_credentials(self, service: str, username: str | None) -> tuple[str, str, bool]: - if keyring is not None: + if keyring.enabled: cred = keyring.get_auth_info(service, username) if cred is not None: return cred[0], cred[1], False @@ -99,7 +101,6 @@ def _prompt_for_credentials(self, service: str, username: str | None) -> tuple[s return username, password, True def _save_credentials(self, service: str, username: str, password: str) -> None: - assert keyring is not None self.ui.echo("Saving credentials to keyring") keyring.save_auth_info(service, username, password) diff --git a/src/pdm/models/auth.py b/src/pdm/models/auth.py index 8909818b95..26cf60bfbb 100644 --- a/src/pdm/models/auth.py +++ b/src/pdm/models/auth.py @@ -53,3 +53,37 @@ def _should_save_password_to_keyring(self) -> bool: style="info", ) return super()._should_save_password_to_keyring() + + +class Keyring: + def __init__(self) -> None: + self.provider = get_keyring_provider() + self.enabled = self.provider is not None + + def get_auth_info(self, url: str, username: str | None) -> tuple[str, str] | None: + """Return the password for the given url and username. + The username can be None. + """ + if self.provider is None: + return None + try: + return self.provider.get_auth_info(url, username) + except Exception: + self.enabled = False + return None + + def save_auth_info(self, url: str, username: str, password: str) -> bool: + """Set the password for the given url and username. + Returns whether the operation is successful. + """ + if self.provider is None: + return False + try: + self.provider.save_auth_info(url, username, password) + return True + except Exception: + self.enabled = False + return False + + +keyring = Keyring() diff --git a/src/pdm/project/config.py b/src/pdm/project/config.py index b717f2764e..7dd8498784 100644 --- a/src/pdm/project/config.py +++ b/src/pdm/project/config.py @@ -299,11 +299,17 @@ def __getitem__(self, key: str) -> Any: return config.coerce(result) def __setitem__(self, key: str, value: Any) -> None: + from pdm.models.auth import keyring + parts = key.split(".") if parts[0] in (REPOSITORY, SOURCE) and key not in self._config_map: if len(parts) < 3: raise PdmUsageError(f"Set {parts[0]} config with [success]{parts[0]}.{{name}}.{{attr}}") index_key = ".".join(parts[:2]) + username = self._data.get(index_key, {}).get("username") + service = f'pdm-{index_key.replace(".", "-")}' + if parts[2] == "password" and self.is_global and keyring.save_auth_info(service, username, value): + return self._file_data.setdefault(index_key, {})[parts[2]] = value self._save_config() return diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 4065a7e549..e963e3d90a 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -13,6 +13,7 @@ from pdm.cli.commands.publish.package import PackageFile from pdm.cli.commands.publish.repository import Repository +from pdm.models.auth import Keyring, keyring from tests import FIXTURES @@ -99,3 +100,29 @@ def _echo(project): """ ) ) + + +@pytest.fixture(name="keyring") +def keyring_fixture(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch) -> Keyring: + from unearth.auth import AuthInfo, KeyringBaseProvider + + class MockKeyringProvider(KeyringBaseProvider): + def __init__(self) -> None: + self._store: dict[str, dict[str, str]] = {} + + def save_auth_info(self, url: str, username: str, password: str) -> None: + self._store.setdefault(url, {})[username] = password + + def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None: + d = self._store.get(url, {}) + if username is not None and username in d: + return username, d[username] + if username is None and d: + return next(iter(d.items())) + return None + + provider = MockKeyringProvider() + mocker.patch("unearth.auth.get_keyring_provider", return_value=provider) + monkeypatch.setattr(keyring, "provider", provider) + monkeypatch.setattr(keyring, "enabled", True) + return keyring diff --git a/tests/cli/test_config.py b/tests/cli/test_config.py index 168ae38f30..d72e223a57 100644 --- a/tests/cli/test_config.py +++ b/tests/cli/test_config.py @@ -185,3 +185,42 @@ def test_config_del_repository(project): del project.global_config["repository.test"] assert project.global_config.get_repository_config("test", "repository") is None + + +def test_config_password_save_into_keyring(project, keyring): + project.global_config.update( + { + "pypi.extra.url": "https://extra.pypi.org/simple", + "pypi.extra.username": "foo", + "pypi.extra.password": "barbaz", + "repository.pypi.username": "frost", + "repository.pypi.password": "password", + } + ) + + assert project.global_config["pypi.extra.password"] == "barbaz" + assert project.global_config["repository.pypi.password"] == "password" + + assert keyring.enabled + assert keyring.get_auth_info("pdm-pypi-extra", "foo") == ("foo", "barbaz") + assert keyring.get_auth_info("pdm-repository-pypi", None) == ("frost", "password") + + +def test_keyring_operation_error_disables_itself(project, keyring, mocker): + mocker.patch.object(keyring.provider, "save_auth_info", side_effect=RuntimeError()) + project.global_config.update( + { + "pypi.extra.url": "https://extra.pypi.org/simple", + "pypi.extra.username": "foo", + "pypi.extra.password": "barbaz", + "repository.pypi.username": "frost", + "repository.pypi.password": "password", + } + ) + + assert project.global_config["pypi.extra.password"] == "barbaz" + assert project.global_config["repository.pypi.password"] == "password" + + assert not keyring.enabled + assert keyring.get_auth_info("pdm-pypi-extra", "foo") is None + assert keyring.get_auth_info("pdm-repository-pypi", None) is None diff --git a/tests/cli/test_publish.py b/tests/cli/test_publish.py index 99f6e62747..cb81501691 100644 --- a/tests/cli/test_publish.py +++ b/tests/cli/test_publish.py @@ -151,3 +151,9 @@ def test_publish_cli_args_and_env_var_precedence(project, monkeypatch): assert repo.url == "https://upload.pypi.org/legacy/" assert repo.session.auth == ("foo", "secret") assert repo.session.verify == "custom.pem" + + +def test_repository_get_credentials_from_keyring(project, keyring): + keyring.save_auth_info("https://test.org/upload", "foo", "barbaz") + repository = Repository(project, "https://test.org/upload", None, None, None) + assert repository.session.auth == ("foo", "barbaz") From 3185d6b6a796bd5a36ddbf9930a415606a64135e Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 11 May 2023 17:09:03 +0800 Subject: [PATCH 2/4] update docs Signed-off-by: Frost Ming --- docs/docs/dev/write.md | 21 +++++++++++++-------- docs/docs/usage/dependency.md | 2 +- docs/docs/usage/scripts.md | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/docs/dev/write.md b/docs/docs/dev/write.md index 4aee0b26ca..8865980a89 100644 --- a/docs/docs/dev/write.md +++ b/docs/docs/dev/write.md @@ -124,21 +124,26 @@ Please read the [API reference](../reference/api.md) for more details. ### Tips about developing a PDM plugin -When developing a plugin, one hopes to activate and plugin in development and get updated when the code changes. This is usually done -by `pip install -e .` or `python setup.py develop` in the **traditional** Python packaging world which leverages `setup.py` to do so. However, -as there is no such `setup.py` in a PDM project, how can we do that? +When developing a plugin, one hopes to activate and plugin in development and get updated when the code changes. -Fortunately, it becomes even easier with PDM and PEP 582. First, you should enable PEP 582 globally following the -[corresponding part of this doc](../usage/pep582.md#enable-pep-582-globally). Then you just need to install all dependencies into the `__pypackages__` directory by: +You can achieve this by installing the plugin in editable mode. To do this, specify the dependencies in `tool.pdm.plugins` array: + +```toml +[tool.pdm] +plugins = [ + "-e file:///{PROJECT_ROOT}" +] +``` + +Then install it with: ```bash -pdm install +pdm install --plugins ``` -After that, all the dependencies are available with a compatible Python interpreter, including the plugin itself, in editable mode. That means any change +After that, all the dependencies are available in a project plugin library, including the plugin itself, in editable mode. That means any change to the codebase will take effect immediately without re-installation. The `pdm` executable also uses a Python interpreter under the hood, so if you run `pdm` from inside the plugin project, the plugin in development will be activated automatically, and you can do some testing to see how it works. -That is how PEP 582 benefits our development workflow. ### Testing your plugin diff --git a/docs/docs/usage/dependency.md b/docs/docs/usage/dependency.md index 9b9c886891..6e78a0faaa 100644 --- a/docs/docs/usage/dependency.md +++ b/docs/docs/usage/dependency.md @@ -286,7 +286,7 @@ dev2 = ["mkdocs"] **All** development dependencies are included as long as `--prod` is not passed and `-G` doesn't specify any dev groups. -Besides, if you don't want the root project to be installed, add `--no-self` option, and `--no-editable` can be used when you want all packages to be installed in non-editable versions. With `--no-editable` turn on, you can safely archive the whole `__pypackages__` and copy it to the target environment for deployment. +Besides, if you don't want the root project to be installed, add `--no-self` option, and `--no-editable` can be used when you want all packages to be installed in non-editable versions. You may also use the pdm lock command with these options to lock only the specified groups, which will be recorded in the `[metadata]` table of the lock file. If no `--group/--prod/--dev/--no-default` option is specified, `pdm sync` and `pdm update` will operate using the groups in the lockfile. However, if any groups that are not included in the lockfile are given as arguments to the commands, PDM will raise an error. diff --git a/docs/docs/usage/scripts.md b/docs/docs/usage/scripts.md index 37a12d57ae..293f40af1f 100644 --- a/docs/docs/usage/scripts.md +++ b/docs/docs/usage/scripts.md @@ -8,7 +8,7 @@ Like `npm run`, with PDM, you can run arbitrary scripts or commands with local p pdm run flask run -p 54321 ``` -It will run `flask run -p 54321` in the environment that is aware of packages in `__pypackages__/` folder. +It will run `flask run -p 54321` in the environment that is aware of packages in your project environment. ## User Scripts From a22ab965b593fb16e953454bfcc403595c634547 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 11 May 2023 19:41:47 +0800 Subject: [PATCH 3/4] update tests Signed-off-by: Frost Ming --- src/pdm/models/auth.py | 4 ++-- tests/cli/test_config.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pdm/models/auth.py b/src/pdm/models/auth.py index 26cf60bfbb..81e11f6182 100644 --- a/src/pdm/models/auth.py +++ b/src/pdm/models/auth.py @@ -64,7 +64,7 @@ def get_auth_info(self, url: str, username: str | None) -> tuple[str, str] | Non """Return the password for the given url and username. The username can be None. """ - if self.provider is None: + if self.provider is None or not self.enabled: return None try: return self.provider.get_auth_info(url, username) @@ -76,7 +76,7 @@ def save_auth_info(self, url: str, username: str, password: str) -> bool: """Set the password for the given url and username. Returns whether the operation is successful. """ - if self.provider is None: + if self.provider is None or not self.enabled: return False try: self.provider.save_auth_info(url, username, password) diff --git a/tests/cli/test_config.py b/tests/cli/test_config.py index d72e223a57..73d1b71db3 100644 --- a/tests/cli/test_config.py +++ b/tests/cli/test_config.py @@ -207,7 +207,8 @@ def test_config_password_save_into_keyring(project, keyring): def test_keyring_operation_error_disables_itself(project, keyring, mocker): - mocker.patch.object(keyring.provider, "save_auth_info", side_effect=RuntimeError()) + saver = mocker.patch.object(keyring.provider, "save_auth_info", side_effect=RuntimeError()) + getter = mocker.patch.object(keyring.provider, "get_auth_info") project.global_config.update( { "pypi.extra.url": "https://extra.pypi.org/simple", @@ -221,6 +222,9 @@ def test_keyring_operation_error_disables_itself(project, keyring, mocker): assert project.global_config["pypi.extra.password"] == "barbaz" assert project.global_config["repository.pypi.password"] == "password" + saver.assert_called_once() + getter.assert_not_called() + assert not keyring.enabled assert keyring.get_auth_info("pdm-pypi-extra", "foo") is None assert keyring.get_auth_info("pdm-repository-pypi", None) is None From eef8746cca889ab7df258a4ee145e3732c67c6fc Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 11 May 2023 19:44:26 +0800 Subject: [PATCH 4/4] restore sys.path after test finish Signed-off-by: Frost Ming --- tests/test_plugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 605396b2e2..4dfae57987 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -1,3 +1,4 @@ +import sys from unittest import mock import pytest @@ -107,7 +108,8 @@ def get_entry_points(group): @pytest.mark.usefixtures("local_finder") -def test_project_plugin_library(pdm, project, core): +def test_project_plugin_library(pdm, project, core, monkeypatch): + monkeypatch.setattr(sys, "path", sys.path[:]) project.pyproject.settings["plugins"] = ["pdm-hello"] pdm(["install", "--plugins"], obj=project, strict=True) assert project.root.joinpath(".pdm-plugins").exists()