From 3207adc804341a8b12063d0267ad2d2c50a891a2 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Tue, 12 Nov 2024 19:43:00 -0500 Subject: [PATCH 01/21] add --with-extra-pip-arg flag --- pex/pip/tool.py | 9 +++++++++ pex/resolve/resolver_configuration.py | 3 ++- pex/resolve/resolver_options.py | 8 ++++++++ pex/resolver.py | 2 ++ tox.ini | 1 + 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 925e560b9..3d281a43b 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -115,6 +115,7 @@ def maybe_trust_insecure_host(url): yield "--timeout" yield str(network_configuration.timeout) + @staticmethod def _calculate_env( network_configuration, # type: NetworkConfiguration @@ -151,6 +152,7 @@ def create( password_entries=(), # type: Iterable[PasswordEntry] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + extra_pip_args=(), # type: Tuple[str, ...] ): # type: (...) -> PackageIndexConfiguration resolver_version = resolver_version or ResolverVersion.default(pip_version) @@ -174,6 +176,7 @@ def create( use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, password_entries=password_entries, + extra_pip_args=extra_pip_args, ) def __init__( @@ -186,6 +189,7 @@ def __init__( password_entries=(), # type: Iterable[PasswordEntry] pip_version=None, # type: Optional[PipVersionValue] extra_pip_requirements=(), # type: Tuple[Requirement, ...] + extra_pip_args=(), # type: Tuple[str, ...] ): # type: (...) -> None self.resolver_version = resolver_version # type: ResolverVersion.Value @@ -196,6 +200,7 @@ def __init__( self.password_entries = password_entries # type: Iterable[PasswordEntry] self.pip_version = pip_version # type: Optional[PipVersionValue] self.extra_pip_requirements = extra_pip_requirements # type: Tuple[Requirement, ...] + self.extra_pip_args = extra_pip_args # type: Tuple[str, ...] if TYPE_CHECKING: @@ -409,6 +414,10 @@ def _spawn_pip_isolated( command = pip_args + list(args) + # Append any pass-through pip args. + if package_index_configuration: + command.extend(package_index_configuration.extra_pip_args) + # N.B.: Package index options in Pep always have the same option names, but they are # registered as subcommand-specific, so we must append them here _after_ the pip subcommand # specified in `args`. diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index ae0ed1c1c..f5263c85a 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -200,7 +200,8 @@ class PipConfiguration(object): allow_version_fallback = attr.ib(default=True) # type: bool use_pip_config = attr.ib(default=False) # type: bool extra_requirements = attr.ib(default=()) # type Tuple[Requirement, ...] - + extra_pip_args = attr.ib(default=()) # type: Tuple[str, ...] + @attr.s(frozen=True) class PexRepositoryConfiguration(object): diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 9517e933a..d7663719e 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -481,6 +481,14 @@ def register_repos_options(parser): type=str, help="Additional cheeseshop indices to use to satisfy requirements.", ) + parser.add_argument( + "--with-extra-pip-arg", + metavar="PIP_ARG", + action="append", + dest="extra_pip_args", + type=str, + help="Additional arguments to pass through to pip.", + ) def register_network_options(parser): diff --git a/pex/resolver.py b/pex/resolver.py index 9dacac413..95a4df3f9 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -1023,6 +1023,7 @@ def resolve( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + extra_pip_args=(), # type: Tuple[str, ...] result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): @@ -1108,6 +1109,7 @@ def resolve( password_entries=password_entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + extra_pip_args=extra_pip_args, ) if not build_configuration.allow_wheels: diff --git a/tox.ini b/tox.ini index 1c28742fa..a62e3dc0e 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ requires = # Ensure tox and virtualenv compatible back through Python 2.7. tox<4 virtualenv<20.16 + setuptools [testenv] # N.B.: We need modern setuptools downloaded out of band by virtualenv to work with Python>=3.12. From c74dc81174bc35147224243f213d1db104ded5c8 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Tue, 12 Nov 2024 23:05:03 -0500 Subject: [PATCH 02/21] revert setuptools addition --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index a62e3dc0e..1c28742fa 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,6 @@ requires = # Ensure tox and virtualenv compatible back through Python 2.7. tox<4 virtualenv<20.16 - setuptools [testenv] # N.B.: We need modern setuptools downloaded out of band by virtualenv to work with Python>=3.12. From 1fc4eb742e3dc0f94168992279cfde3ce00721a4 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 13 Nov 2024 01:02:56 -0500 Subject: [PATCH 03/21] --keychain-provider --- pex/cli/commands/lock.py | 1 + pex/pip/tool.py | 18 ++++++------- pex/resolve/configured_resolve.py | 1 + pex/resolve/configured_resolver.py | 2 ++ pex/resolve/lock_resolver.py | 3 +++ pex/resolve/lockfile/create.py | 1 + pex/resolve/pre_resolved_resolver.py | 1 + pex/resolve/resolver_configuration.py | 4 +-- pex/resolve/resolver_options.py | 19 ++++++++------ pex/resolver.py | 6 +++-- tests/test_pip.py | 38 +++++++++++++++++++++++++-- 11 files changed, 71 insertions(+), 23 deletions(-) diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 7d96aa8a3..69cacfd95 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -1703,6 +1703,7 @@ def _sync(self): pip_version=pip_configuration.version, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keychain_provider=pip_configuration.keychain_provider, result_type=InstallableType.INSTALLED_WHEEL_CHROOT, ) ) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 3d281a43b..f80904185 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -115,7 +115,6 @@ def maybe_trust_insecure_host(url): yield "--timeout" yield str(network_configuration.timeout) - @staticmethod def _calculate_env( network_configuration, # type: NetworkConfiguration @@ -152,7 +151,7 @@ def create( password_entries=(), # type: Iterable[PasswordEntry] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] - extra_pip_args=(), # type: Tuple[str, ...] + keychain_provider=None, # type: Optional[str] ): # type: (...) -> PackageIndexConfiguration resolver_version = resolver_version or ResolverVersion.default(pip_version) @@ -176,7 +175,7 @@ def create( use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, password_entries=password_entries, - extra_pip_args=extra_pip_args, + keychain_provider=keychain_provider, ) def __init__( @@ -189,7 +188,7 @@ def __init__( password_entries=(), # type: Iterable[PasswordEntry] pip_version=None, # type: Optional[PipVersionValue] extra_pip_requirements=(), # type: Tuple[Requirement, ...] - extra_pip_args=(), # type: Tuple[str, ...] + keychain_provider=None, # type: Optional[str] ): # type: (...) -> None self.resolver_version = resolver_version # type: ResolverVersion.Value @@ -200,7 +199,7 @@ def __init__( self.password_entries = password_entries # type: Iterable[PasswordEntry] self.pip_version = pip_version # type: Optional[PipVersionValue] self.extra_pip_requirements = extra_pip_requirements # type: Tuple[Requirement, ...] - self.extra_pip_args = extra_pip_args # type: Tuple[str, ...] + self.keychain_provider = keychain_provider # type: Optional[str] if TYPE_CHECKING: @@ -398,6 +397,11 @@ def _spawn_pip_isolated( # `~/.config/pip/pip.conf`. pip_args.append("--isolated") + # Configure a keychain provider if so configured. + if package_index_configuration and package_index_configuration.keychain_provider: + pip_args.append("--keychain-provider") + pip_args.append(package_index_configuration.keychain_provider) + if log: pip_args.append("--log") pip_args.append(log) @@ -414,10 +418,6 @@ def _spawn_pip_isolated( command = pip_args + list(args) - # Append any pass-through pip args. - if package_index_configuration: - command.extend(package_index_configuration.extra_pip_args) - # N.B.: Package index options in Pep always have the same option names, but they are # registered as subcommand-specific, so we must append them here _after_ the pip subcommand # specified in `args`. diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py index d3b5c11d9..cb8766d74 100644 --- a/pex/resolve/configured_resolve.py +++ b/pex/resolve/configured_resolve.py @@ -64,6 +64,7 @@ def resolve( pip_version=lock.pip_version, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keychain_provider=pip_configuration.keychain_provider, result_type=result_type, dependency_configuration=dependency_configuration, ) diff --git a/pex/resolve/configured_resolver.py b/pex/resolve/configured_resolver.py index 6e76e9e66..24f87538a 100644 --- a/pex/resolve/configured_resolver.py +++ b/pex/resolve/configured_resolver.py @@ -77,6 +77,7 @@ def resolve_lock( pip_version=pip_version or self.pip_configuration.version, use_pip_config=self.pip_configuration.use_pip_config, extra_pip_requirements=self.pip_configuration.extra_requirements, + keychain_provider=self.pip_configuration.keychain_provider, result_type=result_type, ) ) @@ -113,5 +114,6 @@ def resolve_requirements( if extra_resolver_requirements is not None else self.pip_configuration.extra_requirements ), + keychain_provider=self.pip_configuration.keychain_provider, result_type=result_type, ) diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index 1b7f58e47..2b9f2890d 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -257,6 +257,7 @@ def resolve_from_lock( pip_version=None, # type: Optional[PipVersionValue] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keychain_provider=None, # type: Optional[str] result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): @@ -311,6 +312,7 @@ def resolve_from_lock( password_entries=PasswordDatabase.from_netrc().append(password_entries).entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keychain_provider=keychain_provider, ), max_parallel_jobs=max_parallel_jobs, ), @@ -450,6 +452,7 @@ def resolve_from_lock( password_entries=PasswordDatabase.from_netrc().append(password_entries).entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keychain_provider=keychain_provider, ), compile=compile, build_configuration=build_configuration, diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 5d153121c..2f387a04d 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -379,6 +379,7 @@ def create( ), use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keychain_provider=pip_configuration.keychain_provider, ) configured_resolver = ConfiguredResolver(pip_configuration=pip_configuration) diff --git a/pex/resolve/pre_resolved_resolver.py b/pex/resolve/pre_resolved_resolver.py index cf91d923f..19018ef13 100644 --- a/pex/resolve/pre_resolved_resolver.py +++ b/pex/resolve/pre_resolved_resolver.py @@ -102,6 +102,7 @@ def resolve_from_dists( password_entries=pip_configuration.repos_configuration.password_entries, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keychain_provider=pip_configuration.keychain_provider, ) build_and_install = BuildAndInstallRequest( build_requests=[ diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index f5263c85a..d931bd379 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -200,8 +200,8 @@ class PipConfiguration(object): allow_version_fallback = attr.ib(default=True) # type: bool use_pip_config = attr.ib(default=False) # type: bool extra_requirements = attr.ib(default=()) # type Tuple[Requirement, ...] - extra_pip_args = attr.ib(default=()) # type: Tuple[str, ...] - + keychain_provider = attr.ib(default=None) # type: Optional[str] + @attr.s(frozen=True) class PexRepositoryConfiguration(object): diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index d7663719e..cbf29246c 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -135,8 +135,18 @@ def register( "See: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support" ), ) + register_use_pip_config(parser) + parser.add_argument( + "--keychain-provider", + metavar="PROVIDER", + dest="keychain_provider", + type=str, + default=None, + help="Keychain provider to configure `pip` to use.", + ) + register_repos_options(parser) register_network_options(parser) @@ -481,14 +491,6 @@ def register_repos_options(parser): type=str, help="Additional cheeseshop indices to use to satisfy requirements.", ) - parser.add_argument( - "--with-extra-pip-arg", - metavar="PIP_ARG", - action="append", - dest="extra_pip_args", - type=str, - help="Additional arguments to pass through to pip.", - ) def register_network_options(parser): @@ -689,6 +691,7 @@ def create_pip_configuration( allow_version_fallback=options.allow_pip_version_fallback, use_pip_config=get_use_pip_config_value(options), extra_requirements=tuple(options.extra_pip_requirements), + keychain_provider=options.keychain_provider, ) diff --git a/pex/resolver.py b/pex/resolver.py index 95a4df3f9..fcdcb35d6 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -1023,7 +1023,7 @@ def resolve( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] - extra_pip_args=(), # type: Tuple[str, ...] + keychain_provider=None, # type: Optional[str] result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): @@ -1109,7 +1109,7 @@ def resolve( password_entries=password_entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, - extra_pip_args=extra_pip_args, + keychain_provider=keychain_provider, ) if not build_configuration.allow_wheels: @@ -1278,6 +1278,7 @@ def download( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keychain_provider=None, # type: Optional[str] dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): # type: (...) -> Downloaded @@ -1324,6 +1325,7 @@ def download( password_entries=password_entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keychain_provider=keychain_provider, ) build_requests, download_results = _download_internal( targets=targets, diff --git a/tests/test_pip.py b/tests/test_pip.py index ca66c9a3f..5c2f42b1e 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -115,15 +115,21 @@ def test_no_duplicate_constraints_pex_warnings( def package_index_configuration( pip_version, # type: PipVersionValue use_pip_config=False, # type: bool + keychain_provider=None, # type: Optional[str] ): # type: (...) -> PackageIndexConfiguration if pip_version is PipVersion.v23_2: # N.B.: Pip 23.2 has a bug handling PEP-658 metadata with the legacy resolver; so we use the # 2020 resolver to work around. See: https://github.com/pypa/pip/issues/12156 return PackageIndexConfiguration.create( - pip_version, resolver_version=ResolverVersion.PIP_2020, use_pip_config=use_pip_config + pip_version, + resolver_version=ResolverVersion.PIP_2020, + use_pip_config=use_pip_config, + keychain_provider=keychain_provider, ) - return PackageIndexConfiguration.create(use_pip_config=use_pip_config) + return PackageIndexConfiguration.create( + use_pip_config=use_pip_config, keychain_provider=keychain_provider + ) @pytest.mark.skipif( @@ -395,6 +401,34 @@ def test_use_pip_config( assert "invalid --python-version value: 'invalid'" in str(exc.value.stderr) +@applicable_pip_versions +def test_keychain_provider( + create_pip, # type: CreatePip + version, # type: PipVersionValue + current_interpreter, # type: PythonInterpreter + tmpdir, # type: Any +): + # type: (...) -> None + + pip = create_pip(current_interpreter, version=version) + + download_dir = os.path.join(str(tmpdir), "downloads") + assert not os.path.exists(download_dir) + + with ENV.patch(PIP_KEYCHAIN_PROVIDER="invalid") as env, environment_as(**env): + assert "invalid" == os.environ["PIP_KEYCHAIN_PROVIDER"] + job = pip.spawn_download_distributions( + download_dir=download_dir, + requirements=["ansicolors==1.1.8"], + package_index_configuration=package_index_configuration( + pip_version=version, keychain_provider="auto" + ), + ) + assert "--keychain-provider" in job._command, "\n".join(job._command) + with pytest.raises(Job.Error) as exc: + job.wait() + + @applicable_pip_versions def test_extra_pip_requirements_pip_not_allowed( create_pip, # type: CreatePip From 2f217088313d0423891e269161f944a2afe488af Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Thu, 14 Nov 2024 09:14:26 -0500 Subject: [PATCH 04/21] more spots --- pex/resolve/configured_resolve.py | 1 + pex/resolve/lock_resolver.py | 4 ++++ pex/resolve/lockfile/create.py | 1 + 3 files changed, 6 insertions(+) diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py index cb8766d74..fd59cc60d 100644 --- a/pex/resolve/configured_resolve.py +++ b/pex/resolve/configured_resolve.py @@ -131,6 +131,7 @@ def resolve( resolver=ConfiguredResolver(pip_configuration=resolver_configuration), use_pip_config=resolver_configuration.use_pip_config, extra_pip_requirements=resolver_configuration.extra_requirements, + keychain_provider=pip_configuration.keychain_provider, result_type=result_type, dependency_configuration=dependency_configuration, ) diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index 2b9f2890d..68e8623c9 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -95,6 +95,7 @@ def __init__( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keychain_provider=None, # type: Optional[str] ): super(VCSArtifactDownloadManager, self).__init__( pex_root=pex_root, file_lock_style=file_lock_style @@ -117,6 +118,7 @@ def __init__( self._resolver = resolver self._use_pip_config = use_pip_config self._extra_pip_requirements = extra_pip_requirements + self._keychain_provider = keychain_provider def save( self, @@ -143,6 +145,7 @@ def save( resolver=self._resolver, use_pip_config=self._use_pip_config, extra_pip_requirements=self._extra_pip_requirements, + keychain_provider=self._keychain_provider, ) if len(downloaded_vcs.local_distributions) != 1: return Error( @@ -334,6 +337,7 @@ def resolve_from_lock( resolver=resolver, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keychain_provider=keychain_provider, ) for resolved_subset in subset_result.subsets } diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 2f387a04d..6ee1ebedf 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -428,6 +428,7 @@ def create( resolver=configured_resolver, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, + keychain_provider=pip_configuration.keychain_provider, dependency_configuration=dependency_configuration, ) except resolvers.ResolveError as e: From ef3a8f2d2a36384e63d541ca2ab33a3df2445831 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Thu, 14 Nov 2024 09:25:13 -0500 Subject: [PATCH 05/21] fix incorrect variable reference --- pex/resolve/configured_resolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py index fd59cc60d..adf3d91d5 100644 --- a/pex/resolve/configured_resolve.py +++ b/pex/resolve/configured_resolve.py @@ -131,7 +131,7 @@ def resolve( resolver=ConfiguredResolver(pip_configuration=resolver_configuration), use_pip_config=resolver_configuration.use_pip_config, extra_pip_requirements=resolver_configuration.extra_requirements, - keychain_provider=pip_configuration.keychain_provider, + keychain_provider=resolver_configuration.keychain_provider, result_type=result_type, dependency_configuration=dependency_configuration, ) From ccc213a8e44d78005afa385f38ad61a95a72ec9f Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Thu, 14 Nov 2024 11:27:37 -0500 Subject: [PATCH 06/21] name the option correctly --- pex/cli/commands/lock.py | 2 +- pex/pip/tool.py | 14 +++++++------- pex/resolve/configured_resolve.py | 4 ++-- pex/resolve/configured_resolver.py | 4 ++-- pex/resolve/lock_resolver.py | 14 +++++++------- pex/resolve/lockfile/create.py | 4 ++-- pex/resolve/pre_resolved_resolver.py | 2 +- pex/resolve/resolver_configuration.py | 2 +- pex/resolve/resolver_options.py | 6 +++--- pex/resolver.py | 8 ++++---- tests/test_pip.py | 12 ++++++------ 11 files changed, 36 insertions(+), 36 deletions(-) diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 69cacfd95..74918ba05 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -1703,7 +1703,7 @@ def _sync(self): pip_version=pip_configuration.version, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, - keychain_provider=pip_configuration.keychain_provider, + keyring_provider=pip_configuration.keyring_provider, result_type=InstallableType.INSTALLED_WHEEL_CHROOT, ) ) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index f80904185..1c0148687 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -151,7 +151,7 @@ def create( password_entries=(), # type: Iterable[PasswordEntry] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] - keychain_provider=None, # type: Optional[str] + keyring_provider=None, # type: Optional[str] ): # type: (...) -> PackageIndexConfiguration resolver_version = resolver_version or ResolverVersion.default(pip_version) @@ -175,7 +175,7 @@ def create( use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, password_entries=password_entries, - keychain_provider=keychain_provider, + keyring_provider=keyring_provider, ) def __init__( @@ -188,7 +188,7 @@ def __init__( password_entries=(), # type: Iterable[PasswordEntry] pip_version=None, # type: Optional[PipVersionValue] extra_pip_requirements=(), # type: Tuple[Requirement, ...] - keychain_provider=None, # type: Optional[str] + keyring_provider=None, # type: Optional[str] ): # type: (...) -> None self.resolver_version = resolver_version # type: ResolverVersion.Value @@ -199,7 +199,7 @@ def __init__( self.password_entries = password_entries # type: Iterable[PasswordEntry] self.pip_version = pip_version # type: Optional[PipVersionValue] self.extra_pip_requirements = extra_pip_requirements # type: Tuple[Requirement, ...] - self.keychain_provider = keychain_provider # type: Optional[str] + self.keyring_provider = keyring_provider # type: Optional[str] if TYPE_CHECKING: @@ -398,9 +398,9 @@ def _spawn_pip_isolated( pip_args.append("--isolated") # Configure a keychain provider if so configured. - if package_index_configuration and package_index_configuration.keychain_provider: - pip_args.append("--keychain-provider") - pip_args.append(package_index_configuration.keychain_provider) + if package_index_configuration and package_index_configuration.keyring_provider: + pip_args.append("--keyring-provider") + pip_args.append(package_index_configuration.keyring_provider) if log: pip_args.append("--log") diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py index adf3d91d5..537d42480 100644 --- a/pex/resolve/configured_resolve.py +++ b/pex/resolve/configured_resolve.py @@ -64,7 +64,7 @@ def resolve( pip_version=lock.pip_version, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, - keychain_provider=pip_configuration.keychain_provider, + keyring_provider=pip_configuration.keyring_provider, result_type=result_type, dependency_configuration=dependency_configuration, ) @@ -131,7 +131,7 @@ def resolve( resolver=ConfiguredResolver(pip_configuration=resolver_configuration), use_pip_config=resolver_configuration.use_pip_config, extra_pip_requirements=resolver_configuration.extra_requirements, - keychain_provider=resolver_configuration.keychain_provider, + keyring_provider=resolver_configuration.keyring_provider, result_type=result_type, dependency_configuration=dependency_configuration, ) diff --git a/pex/resolve/configured_resolver.py b/pex/resolve/configured_resolver.py index 24f87538a..c8952273b 100644 --- a/pex/resolve/configured_resolver.py +++ b/pex/resolve/configured_resolver.py @@ -77,7 +77,7 @@ def resolve_lock( pip_version=pip_version or self.pip_configuration.version, use_pip_config=self.pip_configuration.use_pip_config, extra_pip_requirements=self.pip_configuration.extra_requirements, - keychain_provider=self.pip_configuration.keychain_provider, + keyring_provider=self.pip_configuration.keyring_provider, result_type=result_type, ) ) @@ -114,6 +114,6 @@ def resolve_requirements( if extra_resolver_requirements is not None else self.pip_configuration.extra_requirements ), - keychain_provider=self.pip_configuration.keychain_provider, + keyring_provider=self.pip_configuration.keyring_provider, result_type=result_type, ) diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index 68e8623c9..64b065a9b 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -95,7 +95,7 @@ def __init__( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] - keychain_provider=None, # type: Optional[str] + keyring_provider=None, # type: Optional[str] ): super(VCSArtifactDownloadManager, self).__init__( pex_root=pex_root, file_lock_style=file_lock_style @@ -118,7 +118,7 @@ def __init__( self._resolver = resolver self._use_pip_config = use_pip_config self._extra_pip_requirements = extra_pip_requirements - self._keychain_provider = keychain_provider + self._keyring_provider = keyring_provider def save( self, @@ -145,7 +145,7 @@ def save( resolver=self._resolver, use_pip_config=self._use_pip_config, extra_pip_requirements=self._extra_pip_requirements, - keychain_provider=self._keychain_provider, + keyring_provider=self._keyring_provider, ) if len(downloaded_vcs.local_distributions) != 1: return Error( @@ -260,7 +260,7 @@ def resolve_from_lock( pip_version=None, # type: Optional[PipVersionValue] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] - keychain_provider=None, # type: Optional[str] + keyring_provider=None, # type: Optional[str] result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): @@ -315,7 +315,7 @@ def resolve_from_lock( password_entries=PasswordDatabase.from_netrc().append(password_entries).entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, - keychain_provider=keychain_provider, + keyring_provider=keyring_provider, ), max_parallel_jobs=max_parallel_jobs, ), @@ -337,7 +337,7 @@ def resolve_from_lock( resolver=resolver, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, - keychain_provider=keychain_provider, + keyring_provider=keyring_provider, ) for resolved_subset in subset_result.subsets } @@ -456,7 +456,7 @@ def resolve_from_lock( password_entries=PasswordDatabase.from_netrc().append(password_entries).entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, - keychain_provider=keychain_provider, + keyring_provider=keyring_provider, ), compile=compile, build_configuration=build_configuration, diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 6ee1ebedf..4a4d33f7d 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -379,7 +379,7 @@ def create( ), use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, - keychain_provider=pip_configuration.keychain_provider, + keyring_provider=pip_configuration.keyring_provider, ) configured_resolver = ConfiguredResolver(pip_configuration=pip_configuration) @@ -428,7 +428,7 @@ def create( resolver=configured_resolver, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, - keychain_provider=pip_configuration.keychain_provider, + keyring_provider=pip_configuration.keyring_provider, dependency_configuration=dependency_configuration, ) except resolvers.ResolveError as e: diff --git a/pex/resolve/pre_resolved_resolver.py b/pex/resolve/pre_resolved_resolver.py index 19018ef13..2f3037783 100644 --- a/pex/resolve/pre_resolved_resolver.py +++ b/pex/resolve/pre_resolved_resolver.py @@ -102,7 +102,7 @@ def resolve_from_dists( password_entries=pip_configuration.repos_configuration.password_entries, use_pip_config=pip_configuration.use_pip_config, extra_pip_requirements=pip_configuration.extra_requirements, - keychain_provider=pip_configuration.keychain_provider, + keyring_provider=pip_configuration.keyring_provider, ) build_and_install = BuildAndInstallRequest( build_requests=[ diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index d931bd379..a2d38dbde 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -200,7 +200,7 @@ class PipConfiguration(object): allow_version_fallback = attr.ib(default=True) # type: bool use_pip_config = attr.ib(default=False) # type: bool extra_requirements = attr.ib(default=()) # type Tuple[Requirement, ...] - keychain_provider = attr.ib(default=None) # type: Optional[str] + keyring_provider = attr.ib(default=None) # type: Optional[str] @attr.s(frozen=True) diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index cbf29246c..2e368ec3d 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -139,9 +139,9 @@ def register( register_use_pip_config(parser) parser.add_argument( - "--keychain-provider", + "--keyring-provider", metavar="PROVIDER", - dest="keychain_provider", + dest="keyring_provider", type=str, default=None, help="Keychain provider to configure `pip` to use.", @@ -691,7 +691,7 @@ def create_pip_configuration( allow_version_fallback=options.allow_pip_version_fallback, use_pip_config=get_use_pip_config_value(options), extra_requirements=tuple(options.extra_pip_requirements), - keychain_provider=options.keychain_provider, + keyring_provider=options.keyring_provider, ) diff --git a/pex/resolver.py b/pex/resolver.py index fcdcb35d6..79297d27e 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -1023,7 +1023,7 @@ def resolve( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] - keychain_provider=None, # type: Optional[str] + keyring_provider=None, # type: Optional[str] result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): @@ -1109,7 +1109,7 @@ def resolve( password_entries=password_entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, - keychain_provider=keychain_provider, + keyring_provider=keyring_provider, ) if not build_configuration.allow_wheels: @@ -1278,7 +1278,7 @@ def download( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] - keychain_provider=None, # type: Optional[str] + keyring_provider=None, # type: Optional[str] dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): # type: (...) -> Downloaded @@ -1325,7 +1325,7 @@ def download( password_entries=password_entries, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, - keychain_provider=keychain_provider, + keyring_provider=keyring_provider, ) build_requests, download_results = _download_internal( targets=targets, diff --git a/tests/test_pip.py b/tests/test_pip.py index 5c2f42b1e..30a2139f0 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -115,7 +115,7 @@ def test_no_duplicate_constraints_pex_warnings( def package_index_configuration( pip_version, # type: PipVersionValue use_pip_config=False, # type: bool - keychain_provider=None, # type: Optional[str] + keyring_provider=None, # type: Optional[str] ): # type: (...) -> PackageIndexConfiguration if pip_version is PipVersion.v23_2: @@ -125,10 +125,10 @@ def package_index_configuration( pip_version, resolver_version=ResolverVersion.PIP_2020, use_pip_config=use_pip_config, - keychain_provider=keychain_provider, + keyring_provider=keyring_provider, ) return PackageIndexConfiguration.create( - use_pip_config=use_pip_config, keychain_provider=keychain_provider + use_pip_config=use_pip_config, keyring_provider=keyring_provider ) @@ -402,7 +402,7 @@ def test_use_pip_config( @applicable_pip_versions -def test_keychain_provider( +def test_keyring_provider( create_pip, # type: CreatePip version, # type: PipVersionValue current_interpreter, # type: PythonInterpreter @@ -421,10 +421,10 @@ def test_keychain_provider( download_dir=download_dir, requirements=["ansicolors==1.1.8"], package_index_configuration=package_index_configuration( - pip_version=version, keychain_provider="auto" + pip_version=version, keyring_provider="auto" ), ) - assert "--keychain-provider" in job._command, "\n".join(job._command) + assert "--keyring-provider" in job._command, "\n".join(job._command) with pytest.raises(Job.Error) as exc: job.wait() From e2d0c26ce7a4142058a1728c8da6af99bd0ed8fd Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Thu, 14 Nov 2024 11:43:04 -0500 Subject: [PATCH 07/21] fix test and comments --- pex/resolve/resolver_options.py | 2 +- tests/test_pip.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 2e368ec3d..8c5ed984f 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -144,7 +144,7 @@ def register( dest="keyring_provider", type=str, default=None, - help="Keychain provider to configure `pip` to use.", + help="`keyring` provider to configure `pip` to use.", ) register_repos_options(parser) diff --git a/tests/test_pip.py b/tests/test_pip.py index 30a2139f0..2efec3a04 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -415,8 +415,8 @@ def test_keyring_provider( download_dir = os.path.join(str(tmpdir), "downloads") assert not os.path.exists(download_dir) - with ENV.patch(PIP_KEYCHAIN_PROVIDER="invalid") as env, environment_as(**env): - assert "invalid" == os.environ["PIP_KEYCHAIN_PROVIDER"] + with ENV.patch(PIP_KEYRING_PROVIDER="invalid") as env, environment_as(**env): + assert "invalid" == os.environ["PIP_KEYRING_PROVIDER"] job = pip.spawn_download_distributions( download_dir=download_dir, requirements=["ansicolors==1.1.8"], @@ -424,7 +424,12 @@ def test_keyring_provider( pip_version=version, keyring_provider="auto" ), ) - assert "--keyring-provider" in job._command, "\n".join(job._command) + keyring_arg = job._command.index("--keyring-provider") + if keyring_arg != -1: + assert job._command[keyring_arg : keyring_arg + 2] == ("--keyring-provider", "auto") + else: + pytest.fail("--keyring-provider was not present in the invoked pip command") + with pytest.raises(Job.Error) as exc: job.wait() From 31662b60251e411e89966cd4df4977a5737384e8 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Tue, 19 Nov 2024 12:20:09 -0500 Subject: [PATCH 08/21] add integration test --- tests/integration/test_keyring_support.py | 33 ++++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_keyring_support.py b/tests/integration/test_keyring_support.py index 3c5a50c80..522bea1bb 100644 --- a/tests/integration/test_keyring_support.py +++ b/tests/integration/test_keyring_support.py @@ -210,6 +210,7 @@ def download_pip_requirements( @skip_if_required_keyring_version_not_supported @keyring_provider_pip_versions +@pytest.mark.parametrize("use_keyring_provider_option", [False, True]) def test_subprocess_provider( proxy, # type: Proxy pip_version, # type: PipVersionValue @@ -218,6 +219,7 @@ def test_subprocess_provider( index_reverse_proxy_target, # type: str devpi_clean_env, # type: Mapping[str, Any] tmpdir, # type: Any + use_keyring_provider_option, # type: bool ): # type: (...) -> None @@ -241,6 +243,13 @@ def test_subprocess_provider( ), ).geturl() ) + + new_path = os.pathsep.join((keyring_venv.path_element, os.environ.get("PATH", os.defpath))) + if use_keyring_provider_option: + env = make_env(PATH=new_path, **devpi_clean_env) + else: + env = make_env(PIP_KEYRING_PROVIDER="subprocess", PATH=new_path, **devpi_clean_env) + run_pex_command( args=[ "--pex-root", @@ -254,25 +263,22 @@ def test_subprocess_provider( find_links, "--pip-version", str(pip_version), - "--use-pip-config", + "--keyring-provider=subprocess" + if use_keyring_provider_option + else "--use-pip-config", "cowsay==5.0", "-c", "cowsay", "--", "Subprocess Auth!", ], - env=make_env( - PIP_KEYRING_PROVIDER="subprocess", - PATH=os.pathsep.join( - (keyring_venv.path_element, os.environ.get("PATH", os.defpath)) - ), - **devpi_clean_env - ), + env=env, ).assert_success(expected_output_re=r"^.*\| Subprocess Auth! \|.*$", re_flags=re.DOTALL) @skip_if_required_keyring_version_not_supported @keyring_provider_pip_versions +@pytest.mark.parametrize("use_keyring_provider_option", [False, True]) def test_import_provider( proxy, # type: Proxy pip_version, # type: PipVersionValue @@ -281,6 +287,7 @@ def test_import_provider( index_reverse_proxy_target, # type: str devpi_clean_env, # type: Mapping[str, Any] tmpdir, # type: Any + use_keyring_provider_option, # type: bool ): # type: (...) -> None @@ -306,6 +313,12 @@ def test_import_provider( netloc="localhost:{port}".format(port=port), ).geturl() ) + + if use_keyring_provider_option: + env = make_env(**devpi_clean_env) + else: + env = make_env(PIP_KEYRING_PROVIDER="import", **devpi_clean_env) + run_pex_command( args=[ "--pex-root", @@ -321,12 +334,12 @@ def test_import_provider( str(keyring_venv.backend.project_name), "--pip-version", str(pip_version), - "--use-pip-config", + "--keyring-provider=import" if use_keyring_provider_option else "--use-pip-config", "cowsay==5.0", "-c", "cowsay", "--", "Import Auth!", ], - env=make_env(PIP_KEYRING_PROVIDER="import", **devpi_clean_env), + env=env, ).assert_success(expected_output_re=r"^.*\| Import Auth! \|.*$", re_flags=re.DOTALL) From 755657e7d4d254cf7c991665de41f771c7f44f6f Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 10 Jan 2025 17:50:29 +0000 Subject: [PATCH 09/21] selected wrong side of the merge --- pex/resolve/lock_downloader.py | 3 + pex/resolve/lock_resolver.py | 193 +-------------------------------- 2 files changed, 4 insertions(+), 192 deletions(-) diff --git a/pex/resolve/lock_downloader.py b/pex/resolve/lock_downloader.py index a89885eb7..bda64077b 100644 --- a/pex/resolve/lock_downloader.py +++ b/pex/resolve/lock_downloader.py @@ -86,6 +86,7 @@ def __init__( resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] ): super(VCSArtifactDownloadManager, self).__init__( pex_root=pex_root, file_lock_style=file_lock_style @@ -108,6 +109,7 @@ def __init__( self._resolver = resolver self._use_pip_config = use_pip_config self._extra_pip_requirements = extra_pip_requirements + self._keyring_provider = keyring_provider def save( self, @@ -134,6 +136,7 @@ def save( resolver=self._resolver, use_pip_config=self._use_pip_config, extra_pip_requirements=self._extra_pip_requirements, + keyring_provider=self._keyring_provider, ) if len(downloaded_vcs.local_distributions) != 1: return Error( diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index 328cf4a0b..965bc6886 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -24,198 +24,7 @@ from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Dict, Iterable, Mapping, Optional, Sequence, Tuple, Union - - import attr # vendor:skip - - from pex.hashing import HintedDigest -else: - from pex.third_party import attr - - -class FileArtifactDownloadManager(DownloadManager[FileArtifact]): - def __init__( - self, - file_lock_style, # type: FileLockStyle.Value - downloader, # type: ArtifactDownloader - pex_root=ENV, # type: Union[str, Variables] - ): - super(FileArtifactDownloadManager, self).__init__( - pex_root=pex_root, file_lock_style=file_lock_style - ) - self._downloader = downloader - - def save( - self, - artifact, # type: FileArtifact - project_name, # type: ProjectName - dest_dir, # type: str - digest, # type: HintedDigest - ): - # type: (...) -> Union[str, Error] - return self._downloader.download(artifact=artifact, dest_dir=dest_dir, digest=digest) - - -class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]): - def __init__( - self, - target, # type: Target - file_lock_style, # type: FileLockStyle.Value - indexes=None, # type: Optional[Sequence[str]] - find_links=None, # type: Optional[Sequence[str]] - resolver_version=None, # type: Optional[ResolverVersion.Value] - network_configuration=None, # type: Optional[NetworkConfiguration] - password_entries=(), # type: Iterable[PasswordEntry] - cache=None, # type: Optional[str] - build_configuration=BuildConfiguration(), # type: BuildConfiguration - pex_root=ENV, # type: Union[str, Variables] - pip_version=None, # type: Optional[PipVersionValue] - resolver=None, # type: Optional[Resolver] - use_pip_config=False, # type: bool - extra_pip_requirements=(), # type: Tuple[Requirement, ...] - keyring_provider=None, # type: Optional[str] - ): - super(VCSArtifactDownloadManager, self).__init__( - pex_root=pex_root, file_lock_style=file_lock_style - ) - self._target = target - self._indexes = indexes - self._find_links = find_links - self._resolver_version = resolver_version - self._network_configuration = network_configuration - self._password_entries = password_entries - self._cache = cache - - # Since a VCSArtifactDownloadManager is only used for VCS requirements, a build is both - # required and preferred by the user. - self._build_configuration = attr.evolve( - build_configuration, allow_builds=True, prefer_older_binary=False - ) - - self._pip_version = pip_version - self._resolver = resolver - self._use_pip_config = use_pip_config - self._extra_pip_requirements = extra_pip_requirements - self._keyring_provider = keyring_provider - - def save( - self, - artifact, # type: VCSArtifact - project_name, # type: ProjectName - dest_dir, # type: str - digest, # type: HintedDigest - ): - # type: (...) -> Union[str, Error] - - requirement = artifact.as_unparsed_requirement(project_name) - downloaded_vcs = resolver.download( - targets=Targets.from_target(self._target), - requirements=[requirement], - transitive=False, - indexes=self._indexes, - find_links=self._find_links, - resolver_version=self._resolver_version, - network_configuration=self._network_configuration, - password_entries=self._password_entries, - build_configuration=self._build_configuration, - max_parallel_jobs=1, - pip_version=self._pip_version, - resolver=self._resolver, - use_pip_config=self._use_pip_config, - extra_pip_requirements=self._extra_pip_requirements, - keyring_provider=self._keyring_provider, - ) - if len(downloaded_vcs.local_distributions) != 1: - return Error( - "Expected 1 artifact for an intransitive download of {requirement}, found " - "{count}:\n" - "{downloads}".format( - requirement=requirement, - count=len(downloaded_vcs.local_distributions), - downloads="\n".join( - "{index}. {download}".format(index=index, download=download.path) - for index, download in enumerate( - downloaded_vcs.local_distributions, start=1 - ) - ), - ) - ) - - local_distribution = downloaded_vcs.local_distributions[0] - filename = os.path.basename(local_distribution.path) - digest_vcs_archive( - archive_path=local_distribution.path, - vcs=artifact.vcs, - digest=digest, - ) - shutil.move(local_distribution.path, os.path.join(dest_dir, filename)) - return filename - - -class LocalProjectDownloadManager(DownloadManager[LocalProjectArtifact]): - def __init__( - self, - target, # type: Target - file_lock_style, # type: FileLockStyle.Value - resolver, # type: Resolver - pip_version=None, # type: Optional[PipVersionValue] - pex_root=ENV, # type: Union[str, Variables] - ): - super(LocalProjectDownloadManager, self).__init__( - pex_root=pex_root, file_lock_style=file_lock_style - ) - self._target = target - self._pip_version = pip_version - self._resolver = resolver - - def save( - self, - artifact, # type: LocalProjectArtifact - project_name, # type: ProjectName - dest_dir, # type: str - digest, # type: HintedDigest - ): - # type: (...) -> Union[str, Error] - source_dir_or_error = digest_local_project( - directory=artifact.directory, - digest=digest, - pip_version=self._pip_version, - target=self._target, - resolver=self._resolver, - dest_dir=dest_dir, - ) - if isinstance(source_dir_or_error, Error): - return source_dir_or_error - return os.path.basename(source_dir_or_error) - - -def download_artifact( - downloadable_artifact_and_target, # type: Tuple[DownloadableArtifact, Target] - file_download_managers_by_target, # type: Mapping[Target, FileArtifactDownloadManager] - vcs_download_managers_by_target, # type: Mapping[Target, VCSArtifactDownloadManager] - local_project_download_managers_by_target, # type: Mapping[Target, LocalProjectDownloadManager] -): - # type: (...) -> Union[DownloadedArtifact, Error] - downloadable_artifact, target = downloadable_artifact_and_target - if isinstance(downloadable_artifact.artifact, VCSArtifact): - return catch( - vcs_download_managers_by_target[target].store, - downloadable_artifact.artifact, - downloadable_artifact.pin.project_name, - ) - - if isinstance(downloadable_artifact.artifact, FileArtifact): - return catch( - file_download_managers_by_target[target].store, - downloadable_artifact.artifact, - downloadable_artifact.pin.project_name, - ) - - return catch( - local_project_download_managers_by_target[target].store, - downloadable_artifact.artifact, - downloadable_artifact.pin.project_name, - ) + from typing import Iterable, Optional, Sequence, Tuple, Union def resolve_from_lock( From 5b8894f4d13bb702a620c25237b709373d973e69 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 10 Jan 2025 22:04:28 -0500 Subject: [PATCH 10/21] more post-merge cleanup for lock_downloader --- pex/resolve/lock_downloader.py | 3 ++ pex/resolve/lock_resolver.py | 82 +++++++--------------------------- 2 files changed, 18 insertions(+), 67 deletions(-) diff --git a/pex/resolve/lock_downloader.py b/pex/resolve/lock_downloader.py index bda64077b..36723ec44 100644 --- a/pex/resolve/lock_downloader.py +++ b/pex/resolve/lock_downloader.py @@ -220,6 +220,7 @@ def create( build_configuration=BuildConfiguration(), # type: BuildConfiguration use_pip_config=False, # type: bool extra_pip_requirements=(), # type: Tuple[Requirement, ...] + keyring_provider=None, # type: Optional[str] ): # type: (...) -> LockDownloader @@ -248,6 +249,7 @@ def create( ), use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ), max_parallel_jobs=max_parallel_jobs, ), @@ -269,6 +271,7 @@ def create( resolver=resolver, use_pip_config=use_pip_config, extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ) for target in targets } diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index 965bc6886..5cac8d77f 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -75,73 +75,21 @@ def resolve_from_lock( for downloadable_artifact in resolved_subset.resolved.downloadable_artifacts ) - # Since the download managers are stored to via a thread pool, we need to use BSD style locks. - # These locks are not as portable as POSIX style locks but work with threading unlike POSIX - # locks which are subject to threading-unaware deadlock detection per the standard. Linux, in - # fact, implements deadlock detection for POSIX locks; so we can run afoul of false EDEADLCK - # errors under the right interleaving of processes and threads and download artifact targets. - file_lock_style = FileLockStyle.BSD - - file_download_managers_by_target = { - resolved_subset.target: FileArtifactDownloadManager( - file_lock_style=file_lock_style, - downloader=ArtifactDownloader( - resolver=resolver, - lock_configuration=LockConfiguration( - style=lock.style, - requires_python=lock.requires_python, - target_systems=lock.target_systems, - ), - target=resolved_subset.target, - package_index_configuration=PackageIndexConfiguration.create( - pip_version=pip_version, - resolver_version=resolver_version, - indexes=indexes, - find_links=find_links, - network_configuration=network_configuration, - password_entries=PasswordDatabase.from_netrc().append(password_entries).entries, - use_pip_config=use_pip_config, - extra_pip_requirements=extra_pip_requirements, - keyring_provider=keyring_provider, - ), - max_parallel_jobs=max_parallel_jobs, - ), - ) - for resolved_subset in subset_result.subsets - } - - vcs_download_managers_by_target = { - resolved_subset.target: VCSArtifactDownloadManager( - target=resolved_subset.target, - file_lock_style=file_lock_style, - indexes=indexes, - find_links=find_links, - resolver_version=resolver_version, - network_configuration=network_configuration, - password_entries=password_entries, - build_configuration=build_configuration, - pip_version=pip_version, - resolver=resolver, - use_pip_config=use_pip_config, - extra_pip_requirements=extra_pip_requirements, - keyring_provider=keyring_provider, - ) - for resolved_subset in subset_result.subsets - } - - local_project_download_managers_by_target = { - resolved_subset.target: LocalProjectDownloadManager( - file_lock_style=file_lock_style, - pip_version=pip_version, - target=resolved_subset.target, - resolver=resolver, - ) - for resolved_subset in subset_result.subsets - } - - max_threads = min( - len(downloadable_artifacts_and_targets) or 1, - min(MAX_PARALLEL_DOWNLOADS, 4 * (max_parallel_jobs or cpu_count() or 1)), + lock_downloader = LockDownloader.create( + targets=tuple(resolved_subset.target for resolved_subset in subset_result.subsets), + lock=lock, + resolver=resolver, + indexes=indexes, + find_links=find_links, + max_parallel_jobs=max_parallel_jobs, + pip_version=pip_version, + resolver_version=resolver_version, + network_configuration=network_configuration, + password_entries=password_entries, + build_configuration=build_configuration, + use_pip_config=use_pip_config, + extra_pip_requirements=extra_pip_requirements, + keyring_provider=keyring_provider, ) with TRACER.timed( "Downloading {url_count} distributions to satisfy {requirement_count} requirements".format( From 8b9a06ed74f9066fbda04054114fca920136feea Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Mon, 13 Jan 2025 12:42:43 -0500 Subject: [PATCH 11/21] do not pass `--keyring-provider` on Pip versions which do not support it --- pex/pip/tool.py | 35 +++++++++++++++++++++-- tests/integration/test_keyring_support.py | 4 +++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 1c0148687..b79fc2264 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -6,6 +6,7 @@ import glob import hashlib +import logging import os import re import subprocess @@ -41,6 +42,9 @@ from pex.variables import ENV from pex.venv.virtualenv import Virtualenv +logger = logging.getLogger(__name__) + + if TYPE_CHECKING: from typing import ( Any, @@ -397,10 +401,35 @@ def _spawn_pip_isolated( # `~/.config/pip/pip.conf`. pip_args.append("--isolated") - # Configure a keychain provider if so configured. + # Configure a keychain provider if so configured and the version of Pip supports the option. + # Warn the user if Pex cannot pass the `--keyring-provider` option and suggest a solution. if package_index_configuration and package_index_configuration.keyring_provider: - pip_args.append("--keyring-provider") - pip_args.append(package_index_configuration.keyring_provider) + if self.version.version >= PipVersion.v23_1.version: + pip_args.append("--keyring-provider") + pip_args.append(package_index_configuration.keyring_provider) + else: + logger.warning( + "".join( + [ + "The PEX_KEYRING_PROVIDER option is set, but Pip v{} does not support ".format( + self.version.version + ), + "the `--keyring-provider` option which is only available in Pip v{} and higher. ".format( + PipVersion.v23_1 + ), + "Pex is ignoring PEX_KEYRING_PROVIDER for this particular Pip invocation.\n\n", + "Note: If this Pex invocation fails, it may be because Pex is trying to use its vendored Pip ", + "v{} to bootstrap a newer Pip version which does support `--keyring-provider`, ".format( + PipVersion.VENDORED.version + ), + "but you configured Pex/Pip to use a Python package index which is not available ", + "without additional authentication.\n\n", + "In that case, please manually create a `find-links` directory with that newer version of Pip, so ", + "that Pex will still be able to install the newer version of Pip from the `find-links` directory (which does ", + "not require authentication).", + ] + ) + ) if log: pip_args.append("--log") diff --git a/tests/integration/test_keyring_support.py b/tests/integration/test_keyring_support.py index 522bea1bb..8696842f1 100644 --- a/tests/integration/test_keyring_support.py +++ b/tests/integration/test_keyring_support.py @@ -244,6 +244,8 @@ def test_subprocess_provider( ).geturl() ) + # If we are testing the `--keyring-provider`option, then do not put the option into the environment + # since it will be passed on the command-line. new_path = os.pathsep.join((keyring_venv.path_element, os.environ.get("PATH", os.defpath))) if use_keyring_provider_option: env = make_env(PATH=new_path, **devpi_clean_env) @@ -314,6 +316,8 @@ def test_import_provider( ).geturl() ) + # If we are testing the `--keyring-provider`option, then do not put the option into the environment + # since it will be passed on the command-line. if use_keyring_provider_option: env = make_env(**devpi_clean_env) else: From a070c687629d9f3e1ac3d46b2aca3b35e987b2f7 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Mon, 13 Jan 2025 16:21:56 -0500 Subject: [PATCH 12/21] improve pip test --- tests/test_pip.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/test_pip.py b/tests/test_pip.py index 0f4eaac35..3cbec0c2b 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -5,6 +5,7 @@ import hashlib import json +import logging import os import re import shutil @@ -407,15 +408,20 @@ def test_keyring_provider( version, # type: PipVersionValue current_interpreter, # type: PythonInterpreter tmpdir, # type: Any + caplog, # type: Any ): # type: (...) -> None + has_keyring_provider_option = version >= PipVersion.v23_1 + pip = create_pip(current_interpreter, version=version) download_dir = os.path.join(str(tmpdir), "downloads") assert not os.path.exists(download_dir) - with ENV.patch(PIP_KEYRING_PROVIDER="invalid") as env, environment_as(**env): + with ENV.patch(PIP_KEYRING_PROVIDER="invalid") as env, environment_as(**env), caplog.at_level( + logging.WARNING + ): assert "invalid" == os.environ["PIP_KEYRING_PROVIDER"] job = pip.spawn_download_distributions( download_dir=download_dir, @@ -424,14 +430,21 @@ def test_keyring_provider( pip_version=version, keyring_provider="auto" ), ) - keyring_arg = job._command.index("--keyring-provider") - if keyring_arg != -1: - assert job._command[keyring_arg : keyring_arg + 2] == ("--keyring-provider", "auto") - else: - pytest.fail("--keyring-provider was not present in the invoked pip command") - with pytest.raises(Job.Error) as exc: + cmd_args = tuple(job._command) + if has_keyring_provider_option: + assert "--keyring-provider" in cmd_args + keyring_arg_index = job._command.index("--keyring-provider") + assert cmd_args[keyring_arg_index : keyring_arg_index + 2] == ( + "--keyring-provider", + "auto", + ) + with pytest.raises(Job.Error) as exc: + job.wait() + else: + assert "--keyring-provider" not in cmd_args job.wait() + assert "does not support the `--keyring-provider` option" in caplog.text @applicable_pip_versions From 5bacd6b23429c88ed589f38910e79ce4955ccdc2 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Mon, 13 Jan 2025 17:32:39 -0500 Subject: [PATCH 13/21] redo warning message --- pex/pip/tool.py | 41 ++++++++++++++++++++--------------------- tests/test_pip.py | 11 +++++++---- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index b79fc2264..56acc5b38 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -11,9 +11,10 @@ import re import subprocess import sys +import textwrap from collections import deque -from pex import targets +from pex import pex_warnings, targets from pex.atomic_directory import atomic_directory from pex.auth import PasswordEntry from pex.cache.dirs import PipPexDir @@ -408,28 +409,26 @@ def _spawn_pip_isolated( pip_args.append("--keyring-provider") pip_args.append(package_index_configuration.keyring_provider) else: - logger.warning( - "".join( - [ - "The PEX_KEYRING_PROVIDER option is set, but Pip v{} does not support ".format( - self.version.version - ), - "the `--keyring-provider` option which is only available in Pip v{} and higher. ".format( - PipVersion.v23_1 - ), - "Pex is ignoring PEX_KEYRING_PROVIDER for this particular Pip invocation.\n\n", - "Note: If this Pex invocation fails, it may be because Pex is trying to use its vendored Pip ", - "v{} to bootstrap a newer Pip version which does support `--keyring-provider`, ".format( - PipVersion.VENDORED.version - ), - "but you configured Pex/Pip to use a Python package index which is not available ", - "without additional authentication.\n\n", - "In that case, please manually create a `find-links` directory with that newer version of Pip, so ", - "that Pex will still be able to install the newer version of Pip from the `find-links` directory (which does ", - "not require authentication).", - ] + warn_msg = textwrap.dedent( + """ + The PIP_KEYRING_PROVIDER option is set, but Pip v{THIS_VERSION} does not support the `--keyring-provider` option + which is only available in Pip v{VERSION_23_1} and higher. Pex is ignoring PIP_KEYRING_PROVIDER for this + particular Pip invocation. + + Note: If this Pex invocation fails, it may be because Pex is trying to use its vendored Pip v{VENDORED_VERSION} + to bootstrap a newer Pip version which does support `--keyring-provider`, but you configured Pex/Pip + to use a Python package index which is not available without additional authentication. + + In that case, please manually create a `find-links` directory with that newer version of Pip, so + that Pex will still be able to install the newer version of Pip from the `find-links` directory (which does + not require authentication). + """.format( + THIS_VERSION=self.version.version, + VERSION_23_1=PipVersion.v23_1, + VENDORED_VERSION=PipVersion.VENDORED.version, ) ) + pex_warnings.warn(warn_msg) if log: pip_args.append("--log") diff --git a/tests/test_pip.py b/tests/test_pip.py index 3cbec0c2b..47492d5bf 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -12,6 +12,7 @@ import warnings from typing import Dict +from pex.pex_warnings import PEXWarning import pytest from pex.common import environment_as, safe_rmtree @@ -419,9 +420,9 @@ def test_keyring_provider( download_dir = os.path.join(str(tmpdir), "downloads") assert not os.path.exists(download_dir) - with ENV.patch(PIP_KEYRING_PROVIDER="invalid") as env, environment_as(**env), caplog.at_level( - logging.WARNING - ): + with ENV.patch(PIP_KEYRING_PROVIDER="invalid") as env, environment_as( + **env + ), warnings.catch_warnings(record=True) as events: assert "invalid" == os.environ["PIP_KEYRING_PROVIDER"] job = pip.spawn_download_distributions( download_dir=download_dir, @@ -444,7 +445,9 @@ def test_keyring_provider( else: assert "--keyring-provider" not in cmd_args job.wait() - assert "does not support the `--keyring-provider` option" in caplog.text + assert len(events) == 1 + assert PEXWarning == events[0].category + assert "does not support the `--keyring-provider` option" in str(events[0].message) @applicable_pip_versions From 8dd50989522dccf5e545fed6707dd3453e550136 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Mon, 13 Jan 2025 17:46:08 -0500 Subject: [PATCH 14/21] soften language and fix import --- pex/pip/tool.py | 6 +++--- tests/test_pip.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 56acc5b38..4ddf11099 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -419,9 +419,9 @@ def _spawn_pip_isolated( to bootstrap a newer Pip version which does support `--keyring-provider`, but you configured Pex/Pip to use a Python package index which is not available without additional authentication. - In that case, please manually create a `find-links` directory with that newer version of Pip, so - that Pex will still be able to install the newer version of Pip from the `find-links` directory (which does - not require authentication). + In that case, you might wish to consider manually creating a `find-links` directory with that newer version + of Pip, so that Pex will still be able to install the newer version of Pip from the `find-links` directory + (which does not require authentication). """.format( THIS_VERSION=self.version.version, VERSION_23_1=PipVersion.v23_1, diff --git a/tests/test_pip.py b/tests/test_pip.py index 47492d5bf..ed45c816c 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -12,7 +12,6 @@ import warnings from typing import Dict -from pex.pex_warnings import PEXWarning import pytest from pex.common import environment_as, safe_rmtree @@ -21,6 +20,7 @@ from pex.jobs import Job from pex.pep_440 import Version from pex.pep_503 import ProjectName +from pex.pex_warnings import PEXWarning from pex.pip.installation import _PIP, PipInstallation, get_pip from pex.pip.tool import PackageIndexConfiguration, Pip from pex.pip.version import PipVersion, PipVersionValue From f903bf8c195f623f7709f21d36e93b98ff19190c Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Mon, 13 Jan 2025 18:05:00 -0500 Subject: [PATCH 15/21] remove unused import --- tests/test_pip.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pip.py b/tests/test_pip.py index ed45c816c..95ff36ffd 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -5,7 +5,6 @@ import hashlib import json -import logging import os import re import shutil From 11c38c9cddde5f72119cf7d36fa8a349a978c3e1 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Mon, 13 Jan 2025 18:28:51 -0500 Subject: [PATCH 16/21] more cleanups --- pex/pip/tool.py | 4 ---- tests/test_pip.py | 1 - 2 files changed, 5 deletions(-) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 4ddf11099..fdf484fdf 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -6,7 +6,6 @@ import glob import hashlib -import logging import os import re import subprocess @@ -43,9 +42,6 @@ from pex.variables import ENV from pex.venv.virtualenv import Virtualenv -logger = logging.getLogger(__name__) - - if TYPE_CHECKING: from typing import ( Any, diff --git a/tests/test_pip.py b/tests/test_pip.py index 95ff36ffd..a8f35bc9a 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -408,7 +408,6 @@ def test_keyring_provider( version, # type: PipVersionValue current_interpreter, # type: PythonInterpreter tmpdir, # type: Any - caplog, # type: Any ): # type: (...) -> None From 352eb4e882ed64e2b532f1ac9d9ff5742d063736 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 15 Jan 2025 09:39:06 -0500 Subject: [PATCH 17/21] better help message --- pex/resolve/resolver_options.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 8c5ed984f..5c6b11a63 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -144,7 +144,14 @@ def register( dest="keyring_provider", type=str, default=None, - help="`keyring` provider to configure `pip` to use.", + help=( + "Configure Pip to use the given keyring provider to obtain authentication for package indexes. " + "You must be using Pip v23.1 or higher in order since keyring support was only added in Pip v23.1. " + "There is obviously a bootstrap issue here if your only available index is secured; " + "so you may need to use an additional --find-links repo or --index that is not " + "secured in order to bootstrap a version of Pip which supports keyring. " + "See: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support" + ) ) register_repos_options(parser) From 69f822d9c5b3ba979dd3bd12bd13955250578689 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 15 Jan 2025 09:42:08 -0500 Subject: [PATCH 18/21] little edit --- pex/resolve/resolver_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 5c6b11a63..618531348 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -146,7 +146,7 @@ def register( default=None, help=( "Configure Pip to use the given keyring provider to obtain authentication for package indexes. " - "You must be using Pip v23.1 or higher in order since keyring support was only added in Pip v23.1. " + "You must be using Pip v23.1 or higher since keyring support was only added in Pip v23.1. " "There is obviously a bootstrap issue here if your only available index is secured; " "so you may need to use an additional --find-links repo or --index that is not " "secured in order to bootstrap a version of Pip which supports keyring. " From 9ae677d3fb1585da232efaa6d2065ca60c172978 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 15 Jan 2025 09:46:08 -0500 Subject: [PATCH 19/21] fmt --- pex/resolve/resolver_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 618531348..cc20218dd 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -151,7 +151,7 @@ def register( "so you may need to use an additional --find-links repo or --index that is not " "secured in order to bootstrap a version of Pip which supports keyring. " "See: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support" - ) + ), ) register_repos_options(parser) From 8be40436f48b81fc58ba9ddfdbe1bafe3d873a10 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 15 Jan 2025 12:39:31 -0500 Subject: [PATCH 20/21] edit version language --- pex/resolve/resolver_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index cc20218dd..4e9e23457 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -146,7 +146,7 @@ def register( default=None, help=( "Configure Pip to use the given keyring provider to obtain authentication for package indexes. " - "You must be using Pip v23.1 or higher since keyring support was only added in Pip v23.1. " + "Please note that keyring support is only available in Pip v23.1 and later versions. " "There is obviously a bootstrap issue here if your only available index is secured; " "so you may need to use an additional --find-links repo or --index that is not " "secured in order to bootstrap a version of Pip which supports keyring. " From 668caa6c7fe5b8e4c8a4bd7d79f7834e6c0a35e4 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 15 Jan 2025 12:45:15 -0500 Subject: [PATCH 21/21] update warning to refer to `--keyring-provider` --- pex/pip/tool.py | 7 ++++--- tests/test_pip.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index fdf484fdf..9e2d42711 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -407,9 +407,9 @@ def _spawn_pip_isolated( else: warn_msg = textwrap.dedent( """ - The PIP_KEYRING_PROVIDER option is set, but Pip v{THIS_VERSION} does not support the `--keyring-provider` option - which is only available in Pip v{VERSION_23_1} and higher. Pex is ignoring PIP_KEYRING_PROVIDER for this - particular Pip invocation. + The --keyring-provider option is set to `{PROVIDER}`, but Pip v{THIS_VERSION} does not support the + `--keyring-provider` option (which is only available in Pip v{VERSION_23_1} and later versions). + Consequently, Pex is ignoring the --keyring-provider option for this particular Pip invocation. Note: If this Pex invocation fails, it may be because Pex is trying to use its vendored Pip v{VENDORED_VERSION} to bootstrap a newer Pip version which does support `--keyring-provider`, but you configured Pex/Pip @@ -419,6 +419,7 @@ def _spawn_pip_isolated( of Pip, so that Pex will still be able to install the newer version of Pip from the `find-links` directory (which does not require authentication). """.format( + PROVIDER=package_index_configuration.keyring_provider, THIS_VERSION=self.version.version, VERSION_23_1=PipVersion.v23_1, VENDORED_VERSION=PipVersion.VENDORED.version, diff --git a/tests/test_pip.py b/tests/test_pip.py index a8f35bc9a..c067d4f40 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -445,7 +445,8 @@ def test_keyring_provider( job.wait() assert len(events) == 1 assert PEXWarning == events[0].category - assert "does not support the `--keyring-provider` option" in str(events[0].message) + message = str(events[0].message).replace("\n", " ") + assert "does not support the `--keyring-provider` option" in message @applicable_pip_versions