diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 942de3eac..740962931 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -1756,6 +1756,7 @@ def _sync(self): pip_version=pip_configuration.version, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keyring_provider=pip_configuration.keyring_provider, result_type=InstallableType.INSTALLED_WHEEL_CHROOT, ) ) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 925e560b9..9e2d42711 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -10,9 +10,10 @@ import re import subprocess import sys +import textwrap from collections import deque -from pex import targets +from pex import pex_warnings, targets from pex.atomic_directory import atomic_directory from pex.auth import PasswordEntry from pex.cache.dirs import PipPexDir @@ -151,6 +152,7 @@ def create( password_entries=(), # type: Iterable[PasswordEntry] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] ): # type: (...) -> PackageIndexConfiguration resolver_version = resolver_version or ResolverVersion.default(pip_version) @@ -174,6 +176,7 @@ def create( use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, password_entries=password_entries, + keyring_provider=keyring_provider, ) def __init__( @@ -186,6 +189,7 @@ def __init__( password_entries=(), # type: Iterable[PasswordEntry] pip_version=None, # type: Optional[PipVersionValue] extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] ): # type: (...) -> None self.resolver_version = resolver_version # type: ResolverVersion.Value @@ -196,6 +200,7 @@ def __init__( self.password_entries = password_entries # type: Iterable[PasswordEntry] self.pip_version = pip_version # type: Optional[PipVersionValue] self.extra_pip_requirements = extra_pip_requirements # type: Tuple[Requirement, ...] + self.keyring_provider = keyring_provider # type: Optional[str] if TYPE_CHECKING: @@ -393,6 +398,35 @@ def _spawn_pip_isolated( # `~/.config/pip/pip.conf`. pip_args.append("--isolated") + # Configure a keychain provider if so configured and the version of Pip supports the option. + # Warn the user if Pex cannot pass the `--keyring-provider` option and suggest a solution. + if package_index_configuration and package_index_configuration.keyring_provider: + if self.version.version >= PipVersion.v23_1.version: + pip_args.append("--keyring-provider") + pip_args.append(package_index_configuration.keyring_provider) + else: + warn_msg = textwrap.dedent( + """ + The --keyring-provider option is set to `{PROVIDER}`, but Pip v{THIS_VERSION} does not support the + `--keyring-provider` option (which is only available in Pip v{VERSION_23_1} and later versions). + Consequently, Pex is ignoring the --keyring-provider option for this particular Pip invocation. + + Note: If this Pex invocation fails, it may be because Pex is trying to use its vendored Pip v{VENDORED_VERSION} + to bootstrap a newer Pip version which does support `--keyring-provider`, but you configured Pex/Pip + to use a Python package index which is not available without additional authentication. + + In that case, you might wish to consider manually creating a `find-links` directory with that newer version + of Pip, so that Pex will still be able to install the newer version of Pip from the `find-links` directory + (which does not require authentication). + """.format( + PROVIDER=package_index_configuration.keyring_provider, + THIS_VERSION=self.version.version, + VERSION_23_1=PipVersion.v23_1, + VENDORED_VERSION=PipVersion.VENDORED.version, + ) + ) + pex_warnings.warn(warn_msg) + if log: pip_args.append("--log") pip_args.append(log) diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py index d3b5c11d9..537d42480 100644 --- a/pex/resolve/configured_resolve.py +++ b/pex/resolve/configured_resolve.py @@ -64,6 +64,7 @@ def resolve( pip_version=lock.pip_version, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keyring_provider=pip_configuration.keyring_provider, result_type=result_type, dependency_configuration=dependency_configuration, ) @@ -130,6 +131,7 @@ def resolve( resolver=ConfiguredResolver(pip_configuration=resolver_configuration), use_pip_config=resolver_configuration.use_pip_config, extra_pip_requirements=resolver_configuration.extra_requirements, + keyring_provider=resolver_configuration.keyring_provider, result_type=result_type, dependency_configuration=dependency_configuration, ) diff --git a/pex/resolve/configured_resolver.py b/pex/resolve/configured_resolver.py index 6e76e9e66..c8952273b 100644 --- a/pex/resolve/configured_resolver.py +++ b/pex/resolve/configured_resolver.py @@ -77,6 +77,7 @@ def resolve_lock( pip_version=pip_version or self.pip_configuration.version, use_pip_config=self.pip_configuration.use_pip_config, extra_pip_requirements=self.pip_configuration.extra_requirements, + keyring_provider=self.pip_configuration.keyring_provider, result_type=result_type, ) ) @@ -113,5 +114,6 @@ def resolve_requirements( if extra_resolver_requirements is not None else self.pip_configuration.extra_requirements ), + keyring_provider=self.pip_configuration.keyring_provider, result_type=result_type, ) diff --git a/pex/resolve/lock_downloader.py b/pex/resolve/lock_downloader.py index a89885eb7..36723ec44 100644 --- a/pex/resolve/lock_downloader.py +++ b/pex/resolve/lock_downloader.py @@ -86,6 +86,7 @@ def __init__( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] ): super(VCSArtifactDownloadManager, self).__init__( pex_root=pex_root, file_lock_style=file_lock_style @@ -108,6 +109,7 @@ def __init__( self._resolver = resolver self._use_pip_config = use_pip_config self._extra_pip_requirements = extra_pip_requirements + self._keyring_provider = keyring_provider def save( self, @@ -134,6 +136,7 @@ def save( resolver=self._resolver, use_pip_config=self._use_pip_config, extra_pip_requirements=self._extra_pip_requirements, + keyring_provider=self._keyring_provider, ) if len(downloaded_vcs.local_distributions) != 1: return Error( @@ -217,6 +220,7 @@ def create( build_configuration=BuildConfiguration(), # type: BuildConfiguration use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] ): # type: (...) -> LockDownloader @@ -245,6 +249,7 @@ def create( ), use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ), max_parallel_jobs=max_parallel_jobs, ), @@ -266,6 +271,7 @@ def create( resolver=resolver, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ) for target in targets } diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index a7277f57c..5cac8d77f 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -47,6 +47,7 @@ def resolve_from_lock( pip_version=None, # type: Optional[PipVersionValue] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): @@ -88,6 +89,7 @@ def resolve_from_lock( build_configuration=build_configuration, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ) with TRACER.timed( "Downloading {url_count} distributions to satisfy {requirement_count} requirements".format( @@ -142,6 +144,7 @@ def resolve_from_lock( password_entries=PasswordDatabase.from_netrc().append(password_entries).entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ), compile=compile, build_configuration=build_configuration, diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 220092665..a9e19863f 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -381,6 +381,7 @@ def create( ), use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keyring_provider=pip_configuration.keyring_provider, ) configured_resolver = ConfiguredResolver(pip_configuration=pip_configuration) @@ -429,6 +430,7 @@ def create( resolver=configured_resolver, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keyring_provider=pip_configuration.keyring_provider, dependency_configuration=dependency_configuration, ) except resolvers.ResolveError as e: diff --git a/pex/resolve/pre_resolved_resolver.py b/pex/resolve/pre_resolved_resolver.py index cf91d923f..2f3037783 100644 --- a/pex/resolve/pre_resolved_resolver.py +++ b/pex/resolve/pre_resolved_resolver.py @@ -102,6 +102,7 @@ def resolve_from_dists( password_entries=pip_configuration.repos_configuration.password_entries, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keyring_provider=pip_configuration.keyring_provider, ) build_and_install = BuildAndInstallRequest( build_requests=[ diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index c7a23102f..94275097f 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -203,6 +203,7 @@ class PipConfiguration(object): allow_version_fallback = attr.ib(default=True) # type: bool use_pip_config = attr.ib(default=False) # type: bool extra_requirements = attr.ib(default=()) # type Tuple[Requirement, ...] + keyring_provider = attr.ib(default=None) # type: Optional[str] @attr.s(frozen=True) diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 9517e933a..4e9e23457 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -135,8 +135,25 @@ def register( "See: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support" ), ) + register_use_pip_config(parser) + parser.add_argument( + "--keyring-provider", + metavar="PROVIDER", + dest="keyring_provider", + type=str, + default=None, + help=( + "Configure Pip to use the given keyring provider to obtain authentication for package indexes. " + "Please note that keyring support is only available in Pip v23.1 and later versions. " + "There is obviously a bootstrap issue here if your only available index is secured; " + "so you may need to use an additional --find-links repo or --index that is not " + "secured in order to bootstrap a version of Pip which supports keyring. " + "See: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support" + ), + ) + register_repos_options(parser) register_network_options(parser) @@ -681,6 +698,7 @@ def create_pip_configuration( allow_version_fallback=options.allow_pip_version_fallback, use_pip_config=get_use_pip_config_value(options), extra_requirements=tuple(options.extra_pip_requirements), + keyring_provider=options.keyring_provider, ) diff --git a/pex/resolver.py b/pex/resolver.py index fdfac75aa..64cee7819 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -1070,6 +1070,7 @@ def resolve( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): @@ -1155,6 +1156,7 @@ def resolve( password_entries=password_entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ) if not build_configuration.allow_wheels: @@ -1323,6 +1325,7 @@ def download( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): # type: (...) -> Downloaded @@ -1369,6 +1372,7 @@ def download( password_entries=password_entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ) build_requests, download_results = _download_internal( targets=targets, diff --git a/tests/integration/test_keyring_support.py b/tests/integration/test_keyring_support.py index 3c5a50c80..8696842f1 100644 --- a/tests/integration/test_keyring_support.py +++ b/tests/integration/test_keyring_support.py @@ -210,6 +210,7 @@ def download_pip_requirements( @skip_if_required_keyring_version_not_supported @keyring_provider_pip_versions +@pytest.mark.parametrize("use_keyring_provider_option", [False, True]) def test_subprocess_provider( proxy, # type: Proxy pip_version, # type: PipVersionValue @@ -218,6 +219,7 @@ def test_subprocess_provider( index_reverse_proxy_target, # type: str devpi_clean_env, # type: Mapping[str, Any] tmpdir, # type: Any + use_keyring_provider_option, # type: bool ): # type: (...) -> None @@ -241,6 +243,15 @@ def test_subprocess_provider( ), ).geturl() ) + + # If we are testing the `--keyring-provider`option, then do not put the option into the environment + # since it will be passed on the command-line. + new_path = os.pathsep.join((keyring_venv.path_element, os.environ.get("PATH", os.defpath))) + if use_keyring_provider_option: + env = make_env(PATH=new_path, **devpi_clean_env) + else: + env = make_env(PIP_KEYRING_PROVIDER="subprocess", PATH=new_path, **devpi_clean_env) + run_pex_command( args=[ "--pex-root", @@ -254,25 +265,22 @@ def test_subprocess_provider( find_links, "--pip-version", str(pip_version), - "--use-pip-config", + "--keyring-provider=subprocess" + if use_keyring_provider_option + else "--use-pip-config", "cowsay==5.0", "-c", "cowsay", "--", "Subprocess Auth!", ], - env=make_env( - PIP_KEYRING_PROVIDER="subprocess", - PATH=os.pathsep.join( - (keyring_venv.path_element, os.environ.get("PATH", os.defpath)) - ), - **devpi_clean_env - ), + env=env, ).assert_success(expected_output_re=r"^.*\| Subprocess Auth! \|.*$", re_flags=re.DOTALL) @skip_if_required_keyring_version_not_supported @keyring_provider_pip_versions +@pytest.mark.parametrize("use_keyring_provider_option", [False, True]) def test_import_provider( proxy, # type: Proxy pip_version, # type: PipVersionValue @@ -281,6 +289,7 @@ def test_import_provider( index_reverse_proxy_target, # type: str devpi_clean_env, # type: Mapping[str, Any] tmpdir, # type: Any + use_keyring_provider_option, # type: bool ): # type: (...) -> None @@ -306,6 +315,14 @@ def test_import_provider( netloc="localhost:{port}".format(port=port), ).geturl() ) + + # If we are testing the `--keyring-provider`option, then do not put the option into the environment + # since it will be passed on the command-line. + if use_keyring_provider_option: + env = make_env(**devpi_clean_env) + else: + env = make_env(PIP_KEYRING_PROVIDER="import", **devpi_clean_env) + run_pex_command( args=[ "--pex-root", @@ -321,12 +338,12 @@ def test_import_provider( str(keyring_venv.backend.project_name), "--pip-version", str(pip_version), - "--use-pip-config", + "--keyring-provider=import" if use_keyring_provider_option else "--use-pip-config", "cowsay==5.0", "-c", "cowsay", "--", "Import Auth!", ], - env=make_env(PIP_KEYRING_PROVIDER="import", **devpi_clean_env), + env=env, ).assert_success(expected_output_re=r"^.*\| Import Auth! \|.*$", re_flags=re.DOTALL) diff --git a/tests/test_pip.py b/tests/test_pip.py index 172ab2378..c067d4f40 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -19,6 +19,7 @@ from pex.jobs import Job from pex.pep_440 import Version from pex.pep_503 import ProjectName +from pex.pex_warnings import PEXWarning from pex.pip.installation import _PIP, PipInstallation, get_pip from pex.pip.tool import PackageIndexConfiguration, Pip from pex.pip.version import PipVersion, PipVersionValue @@ -115,15 +116,21 @@ def test_no_duplicate_constraints_pex_warnings( def package_index_configuration( pip_version, # type: PipVersionValue use_pip_config=False, # type: bool + keyring_provider=None, # type: Optional[str] ): # type: (...) -> PackageIndexConfiguration if pip_version is PipVersion.v23_2: # N.B.: Pip 23.2 has a bug handling PEP-658 metadata with the legacy resolver; so we use the # 2020 resolver to work around. See: https://github.com/pypa/pip/issues/12156 return PackageIndexConfiguration.create( - pip_version, resolver_version=ResolverVersion.PIP_2020, use_pip_config=use_pip_config + pip_version, + resolver_version=ResolverVersion.PIP_2020, + use_pip_config=use_pip_config, + keyring_provider=keyring_provider, ) - return PackageIndexConfiguration.create(use_pip_config=use_pip_config) + return PackageIndexConfiguration.create( + use_pip_config=use_pip_config, keyring_provider=keyring_provider + ) @pytest.mark.skipif( @@ -395,6 +402,53 @@ def test_use_pip_config( assert "invalid --python-version value: 'invalid'" in str(exc.value.stderr) +@applicable_pip_versions +def test_keyring_provider( + create_pip, # type: CreatePip + version, # type: PipVersionValue + current_interpreter, # type: PythonInterpreter + tmpdir, # type: Any +): + # type: (...) -> None + + has_keyring_provider_option = version >= PipVersion.v23_1 + + pip = create_pip(current_interpreter, version=version) + + download_dir = os.path.join(str(tmpdir), "downloads") + assert not os.path.exists(download_dir) + + with ENV.patch(PIP_KEYRING_PROVIDER="invalid") as env, environment_as( + **env + ), warnings.catch_warnings(record=True) as events: + assert "invalid" == os.environ["PIP_KEYRING_PROVIDER"] + job = pip.spawn_download_distributions( + download_dir=download_dir, + requirements=["ansicolors==1.1.8"], + package_index_configuration=package_index_configuration( + pip_version=version, keyring_provider="auto" + ), + ) + + cmd_args = tuple(job._command) + if has_keyring_provider_option: + assert "--keyring-provider" in cmd_args + keyring_arg_index = job._command.index("--keyring-provider") + assert cmd_args[keyring_arg_index : keyring_arg_index + 2] == ( + "--keyring-provider", + "auto", + ) + with pytest.raises(Job.Error) as exc: + job.wait() + else: + assert "--keyring-provider" not in cmd_args + job.wait() + assert len(events) == 1 + assert PEXWarning == events[0].category + message = str(events[0].message).replace("\n", " ") + assert "does not support the `--keyring-provider` option" in message + + @applicable_pip_versions def test_extra_pip_requirements_pip_not_allowed( create_pip, # type: CreatePip