From 6a6263e05a4f2dabde45cc91d83ea7fa7d12d494 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 6 Nov 2023 14:06:48 -0800 Subject: [PATCH 1/4] Implement support for `--exclude `. When excluding a requirement from a PEX, any resolved distribution matching that requirement, as well as any of its transitive dependencies not also needed by non-excluded requirements, are elided from the PEX. At runtime these missing dependencies will not trigger boot resolve errors, but they will cause errors if the modules they would have provided are attempted to be imported. If the intention is to load the modules from the runtime environment, then `--pex-inherit-path` / `PEX_INHERIT_PATH` or `PEX_EXTRA_SYS_PATH` knobs must be used to allow the PEX to see distributions installed in the runtime environment. Clearly, you must know what you're doing to use this option and not encounter runtime errors due to import errors. Be ware! A forthcoming `--provided` option, with similar effects on the PEX contents, will both automatically inherit any needed missing distributions from the runtime environment and require all missing distributions are found; failing fast if they are not. Work towards #2097. --- pex/bin/pex.py | 50 +- pex/dependency_manager.py | 108 ++++ pex/environment.py | 27 +- pex/exclude_configuration.py | 38 ++ pex/pex_info.py | 16 + testing/data/__init__.py | 27 + .../data/locks}/__init__.py | 2 +- testing/data/locks/requests.lock.json | 519 ++++++++++++++++++ .../data/platforms}/__init__.py | 0 .../macosx_10_13_x86_64-cp-36-m.tags.txt | 0 tests/integration/test_excludes.py | 77 +++ tests/test_dependency_manager.py | 155 ++++++ tests/test_platform.py | 3 +- 13 files changed, 1002 insertions(+), 20 deletions(-) create mode 100644 pex/dependency_manager.py create mode 100644 pex/exclude_configuration.py create mode 100644 testing/data/__init__.py rename {tests/data/platforms => testing/data/locks}/__init__.py (51%) create mode 100644 testing/data/locks/requests.lock.json rename {tests/data => testing/data/platforms}/__init__.py (100%) rename {tests => testing}/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt (100%) create mode 100644 tests/integration/test_excludes.py create mode 100644 tests/test_dependency_manager.py diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 263edee88..e8c0a0ee0 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -24,6 +24,7 @@ register_global_arguments, ) from pex.common import die, is_pyc_dir, is_pyc_file, safe_mkdtemp +from pex.dependency_manager import DependencyManager from pex.enum import Enum from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraints @@ -290,6 +291,22 @@ def configure_clp_pex_options(parser): ), ) + group.add_argument( + "--exclude", + dest="excluded", + default=[], + type=str, + action="append", + help=( + "Adds a requirement to exclude from the built PEX. Any distribution included in the " + "PEX's resolve that matches the requirement is excluded from the built PEX along with " + "all of its transitive dependencies that are not also required by other non-excluded " + "distributions. At runtime, the PEX will boot without checking the excluded " + "dependencies are available (say, via `--inherit-path`). This option can be used " + "multiple times." + ), + ) + group.add_argument( "--compile", "--no-compile", @@ -808,11 +825,15 @@ def build_pex( pex_info.strip_pex_env = options.strip_pex_env pex_info.interpreter_constraints = interpreter_constraints - for requirements_pex in options.requirements_pexes: - pex_builder.add_from_requirements_pex(requirements_pex) + dependency_manager = DependencyManager() + with TRACER.timed( + "Adding distributions from pexes: {}".format(" ".join(options.requirements_pexes)) + ): + for requirements_pex in options.requirements_pexes: + dependency_manager.add_from_pex(requirements_pex) with TRACER.timed( - "Resolving distributions ({})".format( + "Resolving distributions for requirements: {}".format( " ".join( itertools.chain.from_iterable( ( @@ -824,22 +845,21 @@ def build_pex( ) ): try: - result = resolve( - targets=targets, - requirement_configuration=requirement_configuration, - resolver_configuration=resolver_configuration, - compile_pyc=options.compile, - ignore_errors=options.ignore_errors, - ) - for installed_dist in result.installed_distributions: - pex_builder.add_distribution( - installed_dist.distribution, fingerprint=installed_dist.fingerprint + dependency_manager.add_from_installed( + resolve( + targets=targets, + requirement_configuration=requirement_configuration, + resolver_configuration=resolver_configuration, + compile_pyc=options.compile, + ignore_errors=options.ignore_errors, ) - for direct_req in installed_dist.direct_requirements: - pex_builder.add_requirement(direct_req) + ) except Unsatisfiable as e: die(str(e)) + with TRACER.timed("Configuring PEX dependencies"): + dependency_manager.configure(pex_builder, excluded=options.excluded) + if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: diff --git a/pex/dependency_manager.py b/pex/dependency_manager.py new file mode 100644 index 000000000..8a2557173 --- /dev/null +++ b/pex/dependency_manager.py @@ -0,0 +1,108 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from collections import defaultdict + +from pex import pex_warnings +from pex.dist_metadata import Requirement +from pex.environment import PEXEnvironment +from pex.exclude_configuration import ExcludeConfiguration +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.orderedset import OrderedSet +from pex.pep_503 import ProjectName +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pex.resolve.resolvers import Installed +from pex.tracer import TRACER +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import DefaultDict, Iterable, Iterator + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s +class DependencyManager(object): + _requirements = attr.ib(factory=OrderedSet) # type: OrderedSet[Requirement] + _distributions = attr.ib(factory=OrderedSet) # type: OrderedSet[FingerprintedDistribution] + + def add_from_pex(self, pex): + # type: (str) -> None + + pex_info = PexInfo.from_pex(pex) + self._requirements.update(Requirement.parse(req) for req in pex_info.requirements) + + pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info) + self._distributions.update(pex_environment.iter_distributions()) + + def add_from_installed(self, installed): + # type: (Installed) -> None + + for installed_dist in installed.installed_distributions: + self._requirements.update(installed_dist.direct_requirements) + self._distributions.add(installed_dist.fingerprinted_distribution) + + def configure( + self, + pex_builder, # type: PEXBuilder + excluded=(), # type: Iterable[str] + ): + # type: (...) -> None + + exclude_configuration = ExcludeConfiguration.create(excluded) + exclude_configuration.configure(pex_builder.info) + + dists_by_project_name = defaultdict( + OrderedSet + ) # type: DefaultDict[ProjectName, OrderedSet[FingerprintedDistribution]] + for dist in self._distributions: + dists_by_project_name[dist.distribution.metadata.project_name].add(dist) + + def iter_non_excluded_distributions(requirements): + # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution] + for req in requirements: + candidate_dists = dists_by_project_name[req.project_name] + for candidate_dist in tuple(candidate_dists): + if candidate_dist.distribution not in req: + continue + candidate_dists.discard(candidate_dist) + + excluded_by = exclude_configuration.excluded_by(candidate_dist.distribution) + if excluded_by: + excludes = " and ".join(map(str, excluded_by)) + TRACER.log( + "Skipping adding {candidate}: excluded by {excludes}".format( + candidate=candidate_dist.distribution, excludes=excludes + ) + ) + for root_req in self._requirements: + if candidate_dist.distribution in root_req: + pex_warnings.warn( + "The distribution {dist} was required by the input requirement " + "{root_req} but excluded by configured excludes: " + "{excludes}".format( + dist=candidate_dist.distribution, + root_req=root_req, + excludes=excludes, + ) + ) + continue + + yield candidate_dist + for dep in iter_non_excluded_distributions( + candidate_dist.distribution.requires() + ): + yield dep + + for fingerprinted_dist in iter_non_excluded_distributions(self._requirements): + pex_builder.add_distribution( + dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint + ) + + for requirement in self._requirements: + pex_builder.add_requirement(requirement) diff --git a/pex/environment.py b/pex/environment.py index cb61b09fa..6cc26ce0f 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -12,6 +12,7 @@ from pex import dist_metadata, pex_warnings, targets from pex.common import pluralize from pex.dist_metadata import Distribution, Requirement +from pex.exclude_configuration import ExcludeConfiguration from pex.fingerprinted_distribution import FingerprintedDistribution from pex.inherit_path import InheritPath from pex.interpreter import PythonInterpreter @@ -348,12 +349,23 @@ def _resolve_requirement( resolved_dists_by_key, # type: MutableMapping[_RequirementKey, FingerprintedDistribution] required, # type: bool required_by=None, # type: Optional[Distribution] + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> Iterator[_DistributionNotFound] requirement_key = _RequirementKey.create(requirement) if requirement_key in resolved_dists_by_key: return + excluded_by = exclude_configuration.excluded_by(requirement) + if excluded_by: + TRACER.log( + "Skipping resolving {requirement}: excluded by {excludes}".format( + requirement=requirement, + excludes=" and ".join(map(str, excluded_by)), + ) + ) + return + available_distributions = [ ranked_dist for ranked_dist in self._available_ranked_dists_by_project_name[ @@ -409,6 +421,7 @@ def _resolve_requirement( resolved_dists_by_key, required, required_by=resolved_distribution.distribution, + exclude_configuration=exclude_configuration, ): yield not_found @@ -502,14 +515,21 @@ def resolve(self): # type: () -> Iterable[Distribution] if self._resolved_dists is None: all_reqs = [Requirement.parse(req) for req in self._pex_info.requirements] + exclude_configuration = ExcludeConfiguration.create(excluded=self._pex_info.excluded) self._resolved_dists = tuple( fingerprinted_distribution.distribution - for fingerprinted_distribution in self.resolve_dists(all_reqs) + for fingerprinted_distribution in self.resolve_dists( + all_reqs, exclude_configuration=exclude_configuration + ) ) return self._resolved_dists - def resolve_dists(self, reqs): - # type: (Iterable[Requirement]) -> Iterable[FingerprintedDistribution] + def resolve_dists( + self, + reqs, # type: Iterable[Requirement] + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration + ): + # type: (...) -> Iterable[FingerprintedDistribution] self._update_candidate_distributions(self.iter_distributions()) @@ -538,6 +558,7 @@ def record_unresolved(dist_not_found): requirement=qualified_req_or_not_found.requirement, required=qualified_req_or_not_found.required, resolved_dists_by_key=resolved_dists_by_key, + exclude_configuration=exclude_configuration, ): record_unresolved(not_found) diff --git a/pex/exclude_configuration.py b/pex/exclude_configuration.py new file mode 100644 index 000000000..e5ccd20d7 --- /dev/null +++ b/pex/exclude_configuration.py @@ -0,0 +1,38 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from typing import Iterable, Tuple, Union + +from pex.dist_metadata import Distribution, Requirement +from pex.pex_info import PexInfo +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Iterable + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s(frozen=True) +class ExcludeConfiguration(object): + @classmethod + def create(cls, excluded): + # type: (Iterable[str]) -> ExcludeConfiguration + return cls(excluded=tuple(Requirement.parse(req) for req in excluded)) + + _excluded = attr.ib(factory=tuple) # type: Tuple[Requirement, ...] + + def configure(self, pex_info): + # type: (PexInfo) -> None + for excluded in self._excluded: + pex_info.add_excluded(excluded) + + def excluded_by(self, item): + # type: (Union[Distribution, Requirement]) -> Iterable[Requirement] + if isinstance(item, Distribution): + return tuple(req for req in self._excluded if item in req) + return tuple(req for req in self._excluded if item.project_name == req.project_name) diff --git a/pex/pex_info.py b/pex/pex_info.py index e99700d4f..6fc9b4320 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -21,6 +21,8 @@ if TYPE_CHECKING: from typing import Any, Dict, Iterable, Mapping, Optional, Text, Tuple, Union + from pex.dist_metadata import Requirement + # N.B.: These are expensive imports and PexInfo is used during PEX bootstrapping which we want # to be as fast as possible. from pex.interpreter import PythonInterpreter @@ -144,6 +146,8 @@ def __init__(self, info=None): raise ValueError("Expected requirements to be a list, got %s" % type(requirements)) self._requirements = OrderedSet(self._parse_requirement_tuple(req) for req in requirements) + self._excluded = OrderedSet(self._pex_info.get("excluded", ())) # type: OrderedSet[str] + def _get_safe(self, key): if key not in self._pex_info: return None @@ -445,6 +449,15 @@ def add_requirement(self, requirement): def requirements(self): return self._requirements + def add_excluded(self, requirement): + # type: (Requirement) -> None + self._excluded.add(str(requirement)) + + @property + def excluded(self): + # type: () -> Iterable[str] + return self._excluded + def add_distribution(self, location, sha): self._distributions[location] = sha @@ -527,12 +540,14 @@ def update(self, other): other.interpreter_constraints ) self._requirements.update(other.requirements) + self._excluded.update(other.excluded) def as_json_dict(self): # type: () -> Dict[str, Any] data = self._pex_info.copy() data["inherit_path"] = self.inherit_path.value data["requirements"] = list(self._requirements) + data["excluded"] = list(self._excluded) data["interpreter_constraints"] = [str(ic) for ic in self.interpreter_constraints] data["distributions"] = self._distributions.copy() return data @@ -541,6 +556,7 @@ def dump(self): # type: (...) -> str data = self.as_json_dict() data["requirements"].sort() + data["excluded"].sort() data["interpreter_constraints"].sort() return json.dumps(data, sort_keys=True) diff --git a/testing/data/__init__.py b/testing/data/__init__.py new file mode 100644 index 000000000..740b3a1ff --- /dev/null +++ b/testing/data/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import pkgutil + + +def load(rel_path): + # type: (str) -> bytes + data = pkgutil.get_data(__name__, rel_path) + if data is None: + raise ValueError( + "No resource found at {rel_path} from package {name}.".format( + rel_path=rel_path, name=__name__ + ) + ) + return data + + +def path(*rel_path): + # type: (*str) -> str + path = os.path.join(os.path.dirname(__file__), *rel_path) + if not os.path.isfile(path): + raise ValueError("No resource found at {path}.".format(path=path)) + return path diff --git a/tests/data/platforms/__init__.py b/testing/data/locks/__init__.py similarity index 51% rename from tests/data/platforms/__init__.py rename to testing/data/locks/__init__.py index 87308fe2b..6227712e5 100644 --- a/tests/data/platforms/__init__.py +++ b/testing/data/locks/__init__.py @@ -1,2 +1,2 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/testing/data/locks/requests.lock.json b/testing/data/locks/requests.lock.json new file mode 100644 index 000000000..d091745b4 --- /dev/null +++ b/testing/data/locks/requests.lock.json @@ -0,0 +1,519 @@ +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9", + "url": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", + "url": "https://files.pythonhosted.org/packages/98/98/c2ff18671db109c9f10ed27f5ef610ae05b73bd876664139cf95bd1429aa/certifi-2023.7.22.tar.gz" + } + ], + "project_name": "certifi", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "2023.7.22" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "url": "https://files.pythonhosted.org/packages/28/76/e6222113b83e3622caa4bb41032d0b1bf785250607392e1b778aca0b8a7d/charset_normalizer-3.3.2-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "url": "https://files.pythonhosted.org/packages/05/31/e1f51c76db7be1d4aef220d29fbfa5dbb4a99165d9833dcbf166753b6dc0/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "url": "https://files.pythonhosted.org/packages/05/8c/eb854996d5fef5e4f33ad56927ad053d04dc820e4a3d39023f35cad72617/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "url": "https://files.pythonhosted.org/packages/07/07/7e554f2bbce3295e191f7e653ff15d55309a9ca40d0362fcdab36f01063c/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "url": "https://files.pythonhosted.org/packages/13/82/83c188028b6f38d39538442dd127dc794c602ae6d45d66c469f4063a4c30/charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "url": "https://files.pythonhosted.org/packages/13/f8/eefae0629fa9260f83b826ee3363e311bb03cfdd518dad1bd10d57cb2d84/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "url": "https://files.pythonhosted.org/packages/16/ea/a9e284aa38cccea06b7056d4cbc7adf37670b1f8a668a312864abf1ff7c6/charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "url": "https://files.pythonhosted.org/packages/19/28/573147271fd041d351b438a5665be8223f1dd92f273713cb882ddafe214c/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "url": "https://files.pythonhosted.org/packages/1e/49/7ab74d4ac537ece3bc3334ee08645e231f39f7d6df6347b29a74b0537103/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "url": "https://files.pythonhosted.org/packages/1f/8d/33c860a7032da5b93382cbe2873261f81467e7b37f4ed91e25fed62fd49b/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "url": "https://files.pythonhosted.org/packages/24/9d/2e3ef673dfd5be0154b20363c5cdcc5606f35666544381bee15af3778239/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "url": "https://files.pythonhosted.org/packages/2a/9d/a6d15bd1e3e2914af5955c8eb15f4071997e7078419328fee93dfd497eb7/charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "url": "https://files.pythonhosted.org/packages/2b/61/095a0aa1a84d1481998b534177c8566fdc50bb1233ea9a0478cd3cc075bd/charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "url": "https://files.pythonhosted.org/packages/2d/dc/9dacba68c9ac0ae781d40e1a0c0058e26302ea0660e574ddf6797a0347f7/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "url": "https://files.pythonhosted.org/packages/2e/37/9223632af0872c86d8b851787f0edd3fe66be4a5378f51242b25212f8374/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "url": "https://files.pythonhosted.org/packages/2e/7d/2259318c202f3d17f3fe6438149b3b9e706d1070fe3fcbb28049730bb25c/charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "url": "https://files.pythonhosted.org/packages/33/95/ef68482e4a6adf781fae8d183fb48d6f2be8facb414f49c90ba6a5149cd1/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "url": "https://files.pythonhosted.org/packages/33/c3/3b96a435c5109dd5b6adc8a59ba1d678b302a97938f032e3770cc84cd354/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "url": "https://files.pythonhosted.org/packages/34/2a/f392457d45e24a0c9bfc012887ed4f3c54bf5d4d05a5deb970ffec4b7fc0/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "url": "https://files.pythonhosted.org/packages/3a/52/9f9d17c3b54dc238de384c4cb5a2ef0e27985b42a0e5cc8e8a31d918d48d/charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "url": "https://files.pythonhosted.org/packages/3d/09/d82fe4a34c5f0585f9ea1df090e2a71eb9bb1e469723053e1ee9f57c16f3/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "url": "https://files.pythonhosted.org/packages/3d/85/5b7416b349609d20611a64718bed383b9251b5a601044550f0c8983b8900/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "url": "https://files.pythonhosted.org/packages/3e/33/21a875a61057165e92227466e54ee076b73af1e21fe1b31f1e292251aa1e/charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "url": "https://files.pythonhosted.org/packages/3f/ba/3f5e7be00b215fa10e13d64b1f6237eb6ebea66676a41b2bcdd09fe74323/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "url": "https://files.pythonhosted.org/packages/40/26/f35951c45070edc957ba40a5b1db3cf60a9dbb1b350c2d5bef03e01e61de/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "url": "https://files.pythonhosted.org/packages/43/05/3bf613e719efe68fb3a77f9c536a389f35b95d75424b96b426a47a45ef1d/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "url": "https://files.pythonhosted.org/packages/44/80/b339237b4ce635b4af1c73742459eee5f97201bd92b2371c53e11958392e/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "url": "https://files.pythonhosted.org/packages/45/59/3d27019d3b447a88fe7e7d004a1e04be220227760264cc41b405e863891b/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "url": "https://files.pythonhosted.org/packages/46/6a/d5c26c41c49b546860cc1acabdddf48b0b3fb2685f4f5617ac59261b44ae/charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "url": "https://files.pythonhosted.org/packages/4f/d1/d547cc26acdb0cc458b152f79b2679d7422f29d41581e6fa907861e88af1/charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "url": "https://files.pythonhosted.org/packages/51/fd/0ee5b1c2860bb3c60236d05b6e4ac240cf702b67471138571dad91bcfed8/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "url": "https://files.pythonhosted.org/packages/53/cd/aa4b8a4d82eeceb872f83237b2d27e43e637cac9ffaef19a1321c3bafb67/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561", + "url": "https://files.pythonhosted.org/packages/54/7f/cad0b328759630814fcf9d804bfabaf47776816ad4ef2e9938b7e1123d04/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "url": "https://files.pythonhosted.org/packages/58/78/a0bc646900994df12e07b4ae5c713f2b3e5998f58b9d3720cce2aa45652f/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "url": "https://files.pythonhosted.org/packages/58/a2/0c63d5d7ffac3104b86631b7f2690058c97bf72d3145c0a9cd4fb90c58c2/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "url": "https://files.pythonhosted.org/packages/5b/ae/ce2c12fcac59cb3860b2e2d76dc405253a4475436b1861d95fe75bdea520/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "url": "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "url": "https://files.pythonhosted.org/packages/66/fe/c7d3da40a66a6bf2920cce0f436fa1f62ee28aaf92f412f0bf3b84c8ad6c/charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "url": "https://files.pythonhosted.org/packages/68/77/02839016f6fbbf808e8b38601df6e0e66c17bbab76dff4613f7511413597/charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "url": "https://files.pythonhosted.org/packages/72/1a/641d5c9f59e6af4c7b53da463d07600a695b9824e20849cb6eea8a627761/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "url": "https://files.pythonhosted.org/packages/74/f1/0d9fe69ac441467b737ba7f48c68241487df2f4522dd7246d9426e7c690e/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "url": "https://files.pythonhosted.org/packages/79/66/8946baa705c588521afe10b2d7967300e49380ded089a62d38537264aece/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "url": "https://files.pythonhosted.org/packages/7b/ef/5eb105530b4da8ae37d506ccfa25057961b7b63d581def6f99165ea89c7e/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "url": "https://files.pythonhosted.org/packages/81/b2/160893421adfa3c45554fb418e321ed342bb10c0a4549e855b2b2a3699cb/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "url": "https://files.pythonhosted.org/packages/8d/b7/9e95102e9a8cce6654b85770794b582dda2921ec1fd924c10fbcf215ad31/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "url": "https://files.pythonhosted.org/packages/91/33/749df346e93d7a30cdcb90cbfdd41a06026317bfbfb62cd68307c1a3c543/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "url": "https://files.pythonhosted.org/packages/91/95/e2cfa7ce962e6c4b59a44a6e19e541c3a0317e543f0e0923f844e8d7d21d/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "url": "https://files.pythonhosted.org/packages/98/69/5d8751b4b670d623aa7a47bef061d69c279e9f922f6705147983aa76c3ce/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "url": "https://files.pythonhosted.org/packages/99/b0/9c365f6d79a9f0f3c379ddb40a256a67aa69c59609608fe7feb6235896e1/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "url": "https://files.pythonhosted.org/packages/9e/ef/cd47a63d3200b232792e361cd67530173a09eb011813478b1c0fb8aa7226/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "url": "https://files.pythonhosted.org/packages/a0/b1/4e72ef73d68ebdd4748f2df97130e8428c4625785f2b6ece31f555590c2d/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "url": "https://files.pythonhosted.org/packages/a2/51/e5023f937d7f307c948ed3e5c29c4b7a3e42ed2ee0b8cdf8f3a706089bf0/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "url": "https://files.pythonhosted.org/packages/a8/31/47d018ef89f95b8aded95c589a77c072c55e94b50a41aa99c0a2008a45a4/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "url": "https://files.pythonhosted.org/packages/a8/6f/4ff299b97da2ed6358154b6eb3a2db67da2ae204e53d205aacb18a7e4f34/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "url": "https://files.pythonhosted.org/packages/b2/62/5a5dcb9a71390a9511a253bde19c9c89e0b20118e41080185ea69fb2c209/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "url": "https://files.pythonhosted.org/packages/b3/c1/ebca8e87c714a6a561cfee063f0655f742e54b8ae6e78151f60ba8708b3a/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "url": "https://files.pythonhosted.org/packages/b8/60/e2f67915a51be59d4539ed189eb0a2b0d292bf79270410746becb32bc2c3/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "url": "https://files.pythonhosted.org/packages/bd/28/7ea29e73eea52c7e15b4b9108d0743fc9e4cc2cdb00d275af1df3d46d360/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "url": "https://files.pythonhosted.org/packages/be/4d/9e370f8281cec2fcc9452c4d1ac513324c32957c5f70c73dd2fa8442a21a/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "url": "https://files.pythonhosted.org/packages/c2/65/52aaf47b3dd616c11a19b1052ce7fa6321250a7a0b975f48d8c366733b9f/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "url": "https://files.pythonhosted.org/packages/c9/7a/6d8767fac16f2c80c7fa9f14e0f53d4638271635c306921844dc0b5fd8a6/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "url": "https://files.pythonhosted.org/packages/cc/94/f7cf5e5134175de79ad2059edf2adce18e0685ebdb9227ff0139975d0e93/charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "url": "https://files.pythonhosted.org/packages/cf/7c/f3b682fa053cc21373c9a839e6beba7705857075686a05c72e0f8c4980ca/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "url": "https://files.pythonhosted.org/packages/d1/2f/0d1efd07c74c52b6886c32a3b906fb8afd2fecf448650e73ecb90a5a27f1/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "url": "https://files.pythonhosted.org/packages/d1/b2/fcedc8255ec42afee97f9e6f0145c734bbe104aac28300214593eb326f1d/charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "url": "https://files.pythonhosted.org/packages/d8/b5/eb705c313100defa57da79277d9207dc8d8e45931035862fa64b625bfead/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "url": "https://files.pythonhosted.org/packages/da/f1/3702ba2a7470666a62fd81c58a4c40be00670e5006a67f4d626e57f013ae/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "url": "https://files.pythonhosted.org/packages/dd/51/68b61b90b24ca35495956b718f35a9756ef7d3dd4b3c1508056fa98d1a1b/charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "url": "https://files.pythonhosted.org/packages/df/3e/a06b18788ca2eb6695c9b22325b6fde7dde0f1d1838b1792a0076f58fe9d/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "url": "https://files.pythonhosted.org/packages/e1/9c/60729bf15dc82e3aaf5f71e81686e42e50715a1399770bcde1a9e43d09db/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "url": "https://files.pythonhosted.org/packages/e4/a6/7ee57823d46331ddc37dd00749c95b0edec2c79b15fc0d6e6efb532e89ac/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "url": "https://files.pythonhosted.org/packages/eb/5c/97d97248af4920bc68687d9c3b3c0f47c910e21a8ff80af4565a576bd2f0/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "url": "https://files.pythonhosted.org/packages/ee/fb/14d30eb4956408ee3ae09ad34299131fb383c47df355ddb428a7331cfa1e/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "url": "https://files.pythonhosted.org/packages/ef/d4/a1d72a8f6aa754fdebe91b848912025d30ab7dced61e9ed8aabbf791ed65/charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "url": "https://files.pythonhosted.org/packages/f2/0e/e06bc07ef4673e4d24dc461333c254586bb759fdd075031539bab6514d07/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "url": "https://files.pythonhosted.org/packages/f6/93/bb6cbeec3bf9da9b2eba458c15966658d1daa8b982c642f81c93ad9b40e1/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "url": "https://files.pythonhosted.org/packages/f6/d3/bfc699ab2c4f9245867060744e8136d359412ff1e5ad93be38a46d160f9d/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "url": "https://files.pythonhosted.org/packages/f7/9d/bcf4a449a438ed6f19790eee543a86a740c77508fbc5ddab210ab3ba3a9a/charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl" + } + ], + "project_name": "charset-normalizer", + "requires_dists": [], + "requires_python": ">=3.7.0", + "version": "3.3.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2", + "url": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "url": "https://files.pythonhosted.org/packages/8b/e1/43beb3d38dba6cb420cefa297822eac205a277ab43e5ba5d5c46faf96438/idna-3.4.tar.gz" + } + ], + "project_name": "idna", + "requires_dists": [], + "requires_python": ">=3.5", + "version": "3.4" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", + "url": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1", + "url": "https://files.pythonhosted.org/packages/9d/be/10918a2eac4ae9f02f6cfe6414b7a155ccd8f7f9d4380d62fd5b955065c3/requests-2.31.0.tar.gz" + } + ], + "project_name": "requests", + "requires_dists": [ + "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"", + "certifi>=2017.4.17", + "chardet<6,>=3.0.2; extra == \"use-chardet-on-py3\"", + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1" + ], + "requires_python": ">=3.7", + "version": "2.31.0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e", + "url": "https://files.pythonhosted.org/packages/d2/b2/b157855192a68541a91ba7b2bbcb91f1b4faa51f8bae38d8005c034be524/urllib3-2.0.7-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", + "url": "https://files.pythonhosted.org/packages/af/47/b215df9f71b4fdba1025fc05a77db2ad243fa0926755a52c5e71659f4e3c/urllib3-2.0.7.tar.gz" + } + ], + "project_name": "urllib3", + "requires_dists": [ + "brotli>=1.0.9; platform_python_implementation == \"CPython\" and extra == \"brotli\"", + "brotlicffi>=0.8.0; platform_python_implementation != \"CPython\" and extra == \"brotli\"", + "certifi; extra == \"secure\"", + "cryptography>=1.9; extra == \"secure\"", + "idna>=2.0.0; extra == \"secure\"", + "pyopenssl>=17.1.0; extra == \"secure\"", + "pysocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"", + "urllib3-secure-extra; extra == \"secure\"", + "zstandard>=0.18.0; extra == \"zstd\"" + ], + "requires_python": ">=3.7", + "version": "2.0.7" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.150", + "pip_version": "23.2", + "prefer_older_binary": false, + "requirements": [ + "requests" + ], + "requires_python": [ + "<3.13,>=3.7" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/tests/data/__init__.py b/testing/data/platforms/__init__.py similarity index 100% rename from tests/data/__init__.py rename to testing/data/platforms/__init__.py diff --git a/tests/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt b/testing/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt similarity index 100% rename from tests/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt rename to testing/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt diff --git a/tests/integration/test_excludes.py b/tests/integration/test_excludes.py new file mode 100644 index 000000000..41d520014 --- /dev/null +++ b/tests/integration/test_excludes.py @@ -0,0 +1,77 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import os.path +import subprocess +from os.path import commonprefix + +import pytest + +from pex.executor import Executor +from pex.pep_503 import ProjectName +from pex.pex import PEX +from pex.typing import TYPE_CHECKING +from pex.venv.virtualenv import Virtualenv +from testing import PY_VER, data, make_env, run_pex_command + +if TYPE_CHECKING: + from typing import Any + + +@pytest.mark.skipif(PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13") +def test_exclude(tmpdir): + # type: (Any) -> None + + requests_lock = data.path("locks", "requests.lock.json") + pex_root = os.path.join(str(tmpdir), "pex_root") + pex = os.path.join(str(tmpdir), "pex") + run_pex_command( + args=[ + "--lock", + requests_lock, + "--exclude", + "certifi", + "-o", + pex, + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + ] + ).assert_success() + + assert ProjectName("certifi") not in frozenset( + dist.metadata.project_name for dist in PEX(pex).resolve() + ) + + # The exclude option is buyer beware. A PEX using this option will not work if the excluded + # distributions carry modules that are, in fact, needed at run time. + requests_cmd = [pex, "-c", "import requests, sys; print(sys.modules['certifi'].__file__)"] + expected_import_error_msg = "ModuleNotFoundError: No module named 'certifi'" + + process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = process.communicate() + assert process.returncode != 0 + + assert expected_import_error_msg in stderr.decode("utf-8"), stderr.decode("utf-8") + + venv_dir = os.path.join(str(tmpdir), "venv") + venv = Virtualenv.create(venv_dir) + pip = venv.install_pip() + + # N.B.: The constraining lock requirement is the one expressed by requests: certifi>=2017.4.17 + # The actual locked version is 2023.7.22; so we stress this crease and use a different, but + # allowed, version. + subprocess.check_call(args=[pip, "install", "certifi==2017.4.17"]) + + # Although the venv has certifi available, a PEX is hermetic by default; so it shouldn't be + # used. + with pytest.raises(Executor.NonZeroExit) as exc: + venv.interpreter.execute(args=requests_cmd) + assert expected_import_error_msg in exc.value.stderr + + # Allowing the `sys.path` to be inherited should allow the certifi hole to be filled in. + _, stdout, _ = venv.interpreter.execute( + args=requests_cmd, env=make_env(PEX_INHERIT_PATH="fallback") + ) + assert venv.site_packages_dir == commonprefix([venv.site_packages_dir, stdout.strip()]) diff --git a/tests/test_dependency_manager.py b/tests/test_dependency_manager.py new file mode 100644 index 000000000..98648f9e4 --- /dev/null +++ b/tests/test_dependency_manager.py @@ -0,0 +1,155 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import hashlib +import os.path +import warnings + +import pytest + +from pex.dependency_manager import DependencyManager +from pex.dist_metadata import DistMetadata, Distribution, Requirement +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.orderedset import OrderedSet +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pex.pex_warnings import PEXWarning +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Tuple + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s(frozen=True) +class DistFactory(object): + install_base_dir = attr.ib() # type: str + + def create( + self, + name, # type: str + *requires # type: str + ): + # type: (...) -> FingerprintedDistribution + fingerprint = hashlib.sha256(name.encode("utf-8")).hexdigest() + location = os.path.join(self.install_base_dir, fingerprint, name) + os.makedirs(location) + return FingerprintedDistribution( + distribution=Distribution( + location=location, + metadata=DistMetadata( + project_name=ProjectName(name), + version=Version("0.1.0"), + requires_dists=tuple(Requirement.parse(req) for req in requires), + ), + ), + fingerprint=fingerprint, + ) + + +@pytest.fixture +def dist_factory(tmpdir): + # type: (Any) -> DistFactory + return DistFactory(os.path.join(str(tmpdir), "installed_wheels")) + + +@attr.s(frozen=True) +class DistGraph(object): + root_reqs = attr.ib() # type: Tuple[Requirement, ...] + dists = attr.ib() # type: Tuple[FingerprintedDistribution, ...] + + def dist(self, name): + # type: (str) -> FingerprintedDistribution + project_name = ProjectName(name) + dists = [ + dist for dist in self.dists if project_name == dist.distribution.metadata.project_name + ] + assert len(dists) == 1, "Expected {name} to match one dist, found {found}".format( + name=name, found=" ".join(map(str, dists)) if dists else "none" + ) + return dists[0] + + +@pytest.fixture +def dist_graph(dist_factory): + # type: (DistFactory) -> DistGraph + + # distA distB <--------\ + # \ / \ | + # v v v | + # distC distD (cycle) + # / \ / | + # V v v | + # distE distF ---------/ + + return DistGraph( + root_reqs=(Requirement.parse("a"), Requirement.parse("b")), + dists=( + dist_factory.create("A", "c"), + dist_factory.create("B", "c", "d"), + dist_factory.create("C", "e", "f"), + dist_factory.create("D", "f"), + dist_factory.create("E"), + dist_factory.create("F", "b"), + ), + ) + + +def test_exclude_root_reqs(dist_graph): + # type: (DistGraph) -> None + + dependency_manager = DependencyManager( + requirements=OrderedSet(dist_graph.root_reqs), distributions=OrderedSet(dist_graph.dists) + ) + + pex_info = PexInfo.default() + pex_builder = PEXBuilder(pex_info=pex_info) + + with warnings.catch_warnings(record=True) as events: + dependency_manager.configure(pex_builder, excluded=["a", "b"]) + assert 2 == len(events) + + warning = events[0] + assert PEXWarning == warning.category + assert ( + "The distribution A 0.1.0 was required by the input requirement a but excluded by " + "configured excludes: a" + ) == str(warning.message) + + warning = events[1] + assert PEXWarning == warning.category + assert ( + "The distribution B 0.1.0 was required by the input requirement b but excluded by " + "configured excludes: b" + ) == str(warning.message) + + pex_builder.freeze() + + assert ["a", "b"] == list(pex_info.requirements) + assert ["a", "b"] == list(pex_info.excluded) + assert {} == pex_info.distributions + + +def test_exclude_complex(dist_graph): + # type: (DistGraph) -> None + + dependency_manager = DependencyManager( + requirements=OrderedSet(dist_graph.root_reqs), distributions=OrderedSet(dist_graph.dists) + ) + + pex_info = PexInfo.default() + pex_builder = PEXBuilder(pex_info=pex_info) + dependency_manager.configure(pex_builder, excluded=["c"]) + pex_builder.freeze() + + assert ["a", "b"] == list(pex_info.requirements) + assert ["c"] == list(pex_info.excluded) + expected_dists = [dist_graph.dist(name) for name in ("A", "B", "D", "F")] + assert { + os.path.basename(dist.location): dist.fingerprint for dist in expected_dists + } == pex_info.distributions diff --git a/tests/test_platform.py b/tests/test_platform.py index 49943758d..178f93249 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -11,6 +11,7 @@ from pex.pep_425 import CompatibilityTags from pex.platforms import Platform from pex.third_party.packaging import tags +from testing import data EXPECTED_BASE = [("py27", "none", "any"), ("py2", "none", "any")] @@ -130,7 +131,7 @@ def test_platform_supported_tags(): # A golden file test. This could break if we upgrade Pip and it upgrades packaging which, from # time to time, corrects omissions in tag sets. - golden_tags = pkgutil.get_data(__name__, "data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt") + golden_tags = data.load("platforms/macosx_10_13_x86_64-cp-36-m.tags.txt") assert golden_tags is not None assert ( CompatibilityTags( From 0c07a0de367c0e2a71c767071a41f9333a060aa2 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 6 Nov 2023 14:35:48 -0800 Subject: [PATCH 2/4] Fixup un-guarded typing import. --- pex/exclude_configuration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pex/exclude_configuration.py b/pex/exclude_configuration.py index e5ccd20d7..b4b22fc4b 100644 --- a/pex/exclude_configuration.py +++ b/pex/exclude_configuration.py @@ -3,14 +3,12 @@ from __future__ import absolute_import -from typing import Iterable, Tuple, Union - from pex.dist_metadata import Distribution, Requirement from pex.pex_info import PexInfo from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Iterable + from typing import Iterable, Tuple, Union import attr # vendor:skip else: From 0e49c563e96f31e1f35c220ea5de2f82bab29739 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 6 Nov 2023 14:56:04 -0800 Subject: [PATCH 3/4] Better wording. --- pex/bin/pex.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index e8c0a0ee0..884f7b305 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -298,12 +298,12 @@ def configure_clp_pex_options(parser): type=str, action="append", help=( - "Adds a requirement to exclude from the built PEX. Any distribution included in the " - "PEX's resolve that matches the requirement is excluded from the built PEX along with " - "all of its transitive dependencies that are not also required by other non-excluded " - "distributions. At runtime, the PEX will boot without checking the excluded " - "dependencies are available (say, via `--inherit-path`). This option can be used " - "multiple times." + "Specifies a requirement to exclude from the built PEX. Any distribution included in " + "the PEX's resolve that matches the requirement is excluded from the built PEX along " + "with all of its transitive dependencies that are not also required by other " + "non-excluded distributions. At runtime, the PEX will boot without checking the " + "excluded dependencies are available (say, via `--inherit-path`). This option can be " + "used multiple times." ), ) From a97fcafc9ad19eeb0d85a956e8682f51407b7063 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 6 Nov 2023 18:30:55 -0800 Subject: [PATCH 4/4] Various fixes suggested by Huon. --- pex/bin/pex.py | 7 +- pex/dependency_manager.py | 14 +++- pex/environment.py | 6 +- pex/pex_builder.py | 16 ----- tests/integration/test_excludes.py | 107 +++++++++++++++++++++++------ tests/test_pex_builder.py | 37 ---------- 6 files changed, 107 insertions(+), 80 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 884f7b305..1d9bab0dc 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -826,11 +826,14 @@ def build_pex( pex_info.interpreter_constraints = interpreter_constraints dependency_manager = DependencyManager() + excluded = list(options.excluded) # type: List[str] + with TRACER.timed( "Adding distributions from pexes: {}".format(" ".join(options.requirements_pexes)) ): for requirements_pex in options.requirements_pexes: - dependency_manager.add_from_pex(requirements_pex) + requirements_pex_info = dependency_manager.add_from_pex(requirements_pex) + excluded.extend(requirements_pex_info.excluded) with TRACER.timed( "Resolving distributions for requirements: {}".format( @@ -858,7 +861,7 @@ def build_pex( die(str(e)) with TRACER.timed("Configuring PEX dependencies"): - dependency_manager.configure(pex_builder, excluded=options.excluded) + dependency_manager.configure(pex_builder, excluded=excluded) if options.entry_point: pex_builder.set_entry_point(options.entry_point) diff --git a/pex/dependency_manager.py b/pex/dependency_manager.py index 8a2557173..1cce6523d 100644 --- a/pex/dependency_manager.py +++ b/pex/dependency_manager.py @@ -32,7 +32,7 @@ class DependencyManager(object): _distributions = attr.ib(factory=OrderedSet) # type: OrderedSet[FingerprintedDistribution] def add_from_pex(self, pex): - # type: (str) -> None + # type: (str) -> PexInfo pex_info = PexInfo.from_pex(pex) self._requirements.update(Requirement.parse(req) for req in pex_info.requirements) @@ -40,6 +40,8 @@ def add_from_pex(self, pex): pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info) self._distributions.update(pex_environment.iter_distributions()) + return pex_info + def add_from_installed(self, installed): # type: (Installed) -> None @@ -63,6 +65,12 @@ def configure( for dist in self._distributions: dists_by_project_name[dist.distribution.metadata.project_name].add(dist) + root_requirements_by_project_name = defaultdict( + OrderedSet + ) # type: DefaultDict[ProjectName, OrderedSet[Requirement]] + for root_req in self._requirements: + root_requirements_by_project_name[root_req.project_name].add(root_req) + def iter_non_excluded_distributions(requirements): # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution] for req in requirements: @@ -80,7 +88,9 @@ def iter_non_excluded_distributions(requirements): candidate=candidate_dist.distribution, excludes=excludes ) ) - for root_req in self._requirements: + for root_req in root_requirements_by_project_name[ + candidate_dist.distribution.metadata.project_name + ]: if candidate_dist.distribution in root_req: pex_warnings.warn( "The distribution {dist} was required by the input requirement " diff --git a/pex/environment.py b/pex/environment.py index 6cc26ce0f..fd08c0186 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -346,10 +346,10 @@ def _evaluate_marker( def _resolve_requirement( self, requirement, # type: Requirement + exclude_configuration, # type: ExcludeConfiguration resolved_dists_by_key, # type: MutableMapping[_RequirementKey, FingerprintedDistribution] required, # type: bool required_by=None, # type: Optional[Distribution] - exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> Iterator[_DistributionNotFound] requirement_key = _RequirementKey.create(requirement) @@ -418,10 +418,10 @@ def _resolve_requirement( for not_found in self._resolve_requirement( dep_requirement, + exclude_configuration, resolved_dists_by_key, required, required_by=resolved_distribution.distribution, - exclude_configuration=exclude_configuration, ): yield not_found @@ -556,9 +556,9 @@ def record_unresolved(dist_not_found): with TRACER.timed("Resolving {}".format(qualified_req_or_not_found.requirement), V=2): for not_found in self._resolve_requirement( requirement=qualified_req_or_not_found.requirement, + exclude_configuration=exclude_configuration, required=qualified_req_or_not_found.required, resolved_dists_by_key=resolved_dists_by_key, - exclude_configuration=exclude_configuration, ): record_unresolved(not_found) diff --git a/pex/pex_builder.py b/pex/pex_builder.py index ee70b1654..1cd69be2e 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -29,7 +29,6 @@ from pex.compiler import Compiler from pex.dist_metadata import Distribution, MetadataError from pex.enum import Enum -from pex.environment import PEXEnvironment from pex.finders import get_entry_point_from_console_script, get_script_from_distributions from pex.interpreter import PythonInterpreter from pex.layout import Layout @@ -374,21 +373,6 @@ def add_requirement(self, req): self._ensure_unfrozen("Adding a requirement") self._pex_info.add_requirement(req) - def add_from_requirements_pex(self, pex): - """Add requirements from an existing pex. - - :param pex: The path to an existing .pex file or unzipped pex directory. - """ - self._ensure_unfrozen("Adding from pex") - pex_info = PexInfo.from_pex(pex) - pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info) - for fingerprinted_dist in pex_environment.iter_distributions(): - self.add_distribution( - dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint - ) - for requirement in pex_info.requirements: - self.add_requirement(requirement) - def set_executable(self, filename, env_filename=None): """Set the executable for this environment. diff --git a/tests/integration/test_excludes.py b/tests/integration/test_excludes.py index 41d520014..9896b51df 100644 --- a/tests/integration/test_excludes.py +++ b/tests/integration/test_excludes.py @@ -10,21 +10,22 @@ from pex.executor import Executor from pex.pep_503 import ProjectName from pex.pex import PEX +from pex.pex_info import PexInfo from pex.typing import TYPE_CHECKING from pex.venv.virtualenv import Virtualenv from testing import PY_VER, data, make_env, run_pex_command if TYPE_CHECKING: - from typing import Any + from typing import Any, Tuple -@pytest.mark.skipif(PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13") -def test_exclude(tmpdir): - # type: (Any) -> None +@pytest.fixture(scope="module") +def requests_certifi_excluded_pex(tmpdir_factory): + # type: (Any) -> str requests_lock = data.path("locks", "requests.lock.json") - pex_root = os.path.join(str(tmpdir), "pex_root") - pex = os.path.join(str(tmpdir), "pex") + pex_root = str(tmpdir_factory.mktemp("pex_root")) + pex = str(tmpdir_factory.mktemp("pex")) run_pex_command( args=[ "--lock", @@ -44,19 +45,18 @@ def test_exclude(tmpdir): dist.metadata.project_name for dist in PEX(pex).resolve() ) - # The exclude option is buyer beware. A PEX using this option will not work if the excluded - # distributions carry modules that are, in fact, needed at run time. - requests_cmd = [pex, "-c", "import requests, sys; print(sys.modules['certifi'].__file__)"] - expected_import_error_msg = "ModuleNotFoundError: No module named 'certifi'" + return pex + + +REQUESTS_CMD = ["-c", "import requests, sys; print(sys.modules['certifi'].__file__)"] +EXPECTED_IMPORT_ERROR_MSG = "ModuleNotFoundError: No module named 'certifi'" - process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _, stderr = process.communicate() - assert process.returncode != 0 - assert expected_import_error_msg in stderr.decode("utf-8"), stderr.decode("utf-8") +@pytest.fixture(scope="module") +def certifi_venv(tmpdir_factory): + # type: (Any) -> Virtualenv - venv_dir = os.path.join(str(tmpdir), "venv") - venv = Virtualenv.create(venv_dir) + venv = Virtualenv.create(venv_dir=str(tmpdir_factory.mktemp("venv"))) pip = venv.install_pip() # N.B.: The constraining lock requirement is the one expressed by requests: certifi>=2017.4.17 @@ -64,14 +64,81 @@ def test_exclude(tmpdir): # allowed, version. subprocess.check_call(args=[pip, "install", "certifi==2017.4.17"]) + return venv + + +skip_unless_37_to_312 = pytest.mark.skipif( + PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13" +) + + +def assert_certifi_import_behavior( + pex, # type: str + certifi_venv, # type: Virtualenv +): + requests_cmd = [pex] + REQUESTS_CMD + # Although the venv has certifi available, a PEX is hermetic by default; so it shouldn't be # used. with pytest.raises(Executor.NonZeroExit) as exc: - venv.interpreter.execute(args=requests_cmd) - assert expected_import_error_msg in exc.value.stderr + certifi_venv.interpreter.execute(args=requests_cmd) + assert EXPECTED_IMPORT_ERROR_MSG in exc.value.stderr # Allowing the `sys.path` to be inherited should allow the certifi hole to be filled in. - _, stdout, _ = venv.interpreter.execute( + _, stdout, _ = certifi_venv.interpreter.execute( args=requests_cmd, env=make_env(PEX_INHERIT_PATH="fallback") ) - assert venv.site_packages_dir == commonprefix([venv.site_packages_dir, stdout.strip()]) + assert certifi_venv.site_packages_dir == commonprefix( + [certifi_venv.site_packages_dir, stdout.strip()] + ) + + +@skip_unless_37_to_312 +def test_exclude( + tmpdir, # type: Any + requests_certifi_excluded_pex, # type: str + certifi_venv, # type: Virtualenv +): + # type: (...) -> None + + requests_cmd = [requests_certifi_excluded_pex] + REQUESTS_CMD + + # The exclude option is buyer beware. A PEX using this option will not work if the excluded + # distributions carry modules that are, in fact, needed at run time. + process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = process.communicate() + assert process.returncode != 0 + assert EXPECTED_IMPORT_ERROR_MSG in stderr.decode("utf-8"), stderr.decode("utf-8") + + assert_certifi_import_behavior(requests_certifi_excluded_pex, certifi_venv) + + +@skip_unless_37_to_312 +def test_requirements_pex_exclude( + tmpdir, # type: Any + requests_certifi_excluded_pex, # type: str + certifi_venv, # type: Virtualenv +): + # type: (...) -> None + + pex_root = PexInfo.from_pex(requests_certifi_excluded_pex).pex_root + pex = os.path.join(str(tmpdir), "pex") + run_pex_command( + args=[ + "--requirements-pex", + requests_certifi_excluded_pex, + "ansicolors==1.1.8", + "-o", + pex, + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + ] + ).assert_success() + + # Shouldn't need the certifi hole filled to import colors. + output = subprocess.check_output(args=[pex, "-c", "import colors; print(colors.__file__)"]) + assert pex_root == commonprefix([pex_root, output.decode("utf-8").strip()]) + + assert_certifi_import_behavior(pex, certifi_venv) diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index cafaf2ebb..e99abe05b 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -210,43 +210,6 @@ def test_pex_builder_deterministic_timestamp(): assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist()) -def test_pex_builder_from_requirements_pex(): - # type: () -> None - def build_from_req_pex(path, req_pex): - # type: (str, str) -> PEXBuilder - pb = PEXBuilder(path=path) - pb.add_from_requirements_pex(req_pex) - with open(os.path.join(path, "exe.py"), "w") as fp: - fp.write(exe_main) - pb.set_executable(os.path.join(path, "exe.py")) - pb.freeze() - return pb - - def verify(pb): - # type: (PEXBuilder) -> None - success_txt = os.path.join(pb.path(), "success.txt") - PEX(pb.path(), interpreter=pb.interpreter).run(args=[success_txt]) - assert os.path.exists(success_txt) - with open(success_txt) as fp: - assert fp.read() == "success" - - # Build from pex dir. - with temporary_dir() as td2: - with temporary_dir() as td1, make_bdist("p1") as p1: - pb1 = write_pex(td1, dists=[p1]) - pb2 = build_from_req_pex(td2, pb1.path()) - verify(pb2) - - # Build from .pex file. - with temporary_dir() as td4: - with temporary_dir() as td3, make_bdist("p1") as p1: - pb3 = write_pex(td3, dists=[p1]) - target = os.path.join(td3, "foo.pex") - pb3.build(target) - pb4 = build_from_req_pex(td4, target) - verify(pb4) - - def test_pex_builder_script_from_pex_path(tmpdir): # type: (Any) -> None