diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 263edee88..1d9bab0dc 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -24,6 +24,7 @@ register_global_arguments, ) from pex.common import die, is_pyc_dir, is_pyc_file, safe_mkdtemp +from pex.dependency_manager import DependencyManager from pex.enum import Enum from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraints @@ -290,6 +291,22 @@ def configure_clp_pex_options(parser): ), ) + group.add_argument( + "--exclude", + dest="excluded", + default=[], + type=str, + action="append", + help=( + "Specifies a requirement to exclude from the built PEX. Any distribution included in " + "the PEX's resolve that matches the requirement is excluded from the built PEX along " + "with all of its transitive dependencies that are not also required by other " + "non-excluded distributions. At runtime, the PEX will boot without checking the " + "excluded dependencies are available (say, via `--inherit-path`). This option can be " + "used multiple times." + ), + ) + group.add_argument( "--compile", "--no-compile", @@ -808,11 +825,18 @@ def build_pex( pex_info.strip_pex_env = options.strip_pex_env pex_info.interpreter_constraints = interpreter_constraints - for requirements_pex in options.requirements_pexes: - pex_builder.add_from_requirements_pex(requirements_pex) + dependency_manager = DependencyManager() + excluded = list(options.excluded) # type: List[str] with TRACER.timed( - "Resolving distributions ({})".format( + "Adding distributions from pexes: {}".format(" ".join(options.requirements_pexes)) + ): + for requirements_pex in options.requirements_pexes: + requirements_pex_info = dependency_manager.add_from_pex(requirements_pex) + excluded.extend(requirements_pex_info.excluded) + + with TRACER.timed( + "Resolving distributions for requirements: {}".format( " ".join( itertools.chain.from_iterable( ( @@ -824,22 +848,21 @@ def build_pex( ) ): try: - result = resolve( - targets=targets, - requirement_configuration=requirement_configuration, - resolver_configuration=resolver_configuration, - compile_pyc=options.compile, - ignore_errors=options.ignore_errors, - ) - for installed_dist in result.installed_distributions: - pex_builder.add_distribution( - installed_dist.distribution, fingerprint=installed_dist.fingerprint + dependency_manager.add_from_installed( + resolve( + targets=targets, + requirement_configuration=requirement_configuration, + resolver_configuration=resolver_configuration, + compile_pyc=options.compile, + ignore_errors=options.ignore_errors, ) - for direct_req in installed_dist.direct_requirements: - pex_builder.add_requirement(direct_req) + ) except Unsatisfiable as e: die(str(e)) + with TRACER.timed("Configuring PEX dependencies"): + dependency_manager.configure(pex_builder, excluded=excluded) + if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: diff --git a/pex/dependency_manager.py b/pex/dependency_manager.py new file mode 100644 index 000000000..1cce6523d --- /dev/null +++ b/pex/dependency_manager.py @@ -0,0 +1,118 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from collections import defaultdict + +from pex import pex_warnings +from pex.dist_metadata import Requirement +from pex.environment import PEXEnvironment +from pex.exclude_configuration import ExcludeConfiguration +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.orderedset import OrderedSet +from pex.pep_503 import ProjectName +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pex.resolve.resolvers import Installed +from pex.tracer import TRACER +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import DefaultDict, Iterable, Iterator + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s +class DependencyManager(object): + _requirements = attr.ib(factory=OrderedSet) # type: OrderedSet[Requirement] + _distributions = attr.ib(factory=OrderedSet) # type: OrderedSet[FingerprintedDistribution] + + def add_from_pex(self, pex): + # type: (str) -> PexInfo + + pex_info = PexInfo.from_pex(pex) + self._requirements.update(Requirement.parse(req) for req in pex_info.requirements) + + pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info) + self._distributions.update(pex_environment.iter_distributions()) + + return pex_info + + def add_from_installed(self, installed): + # type: (Installed) -> None + + for installed_dist in installed.installed_distributions: + self._requirements.update(installed_dist.direct_requirements) + self._distributions.add(installed_dist.fingerprinted_distribution) + + def configure( + self, + pex_builder, # type: PEXBuilder + excluded=(), # type: Iterable[str] + ): + # type: (...) -> None + + exclude_configuration = ExcludeConfiguration.create(excluded) + exclude_configuration.configure(pex_builder.info) + + dists_by_project_name = defaultdict( + OrderedSet + ) # type: DefaultDict[ProjectName, OrderedSet[FingerprintedDistribution]] + for dist in self._distributions: + dists_by_project_name[dist.distribution.metadata.project_name].add(dist) + + root_requirements_by_project_name = defaultdict( + OrderedSet + ) # type: DefaultDict[ProjectName, OrderedSet[Requirement]] + for root_req in self._requirements: + root_requirements_by_project_name[root_req.project_name].add(root_req) + + def iter_non_excluded_distributions(requirements): + # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution] + for req in requirements: + candidate_dists = dists_by_project_name[req.project_name] + for candidate_dist in tuple(candidate_dists): + if candidate_dist.distribution not in req: + continue + candidate_dists.discard(candidate_dist) + + excluded_by = exclude_configuration.excluded_by(candidate_dist.distribution) + if excluded_by: + excludes = " and ".join(map(str, excluded_by)) + TRACER.log( + "Skipping adding {candidate}: excluded by {excludes}".format( + candidate=candidate_dist.distribution, excludes=excludes + ) + ) + for root_req in root_requirements_by_project_name[ + candidate_dist.distribution.metadata.project_name + ]: + if candidate_dist.distribution in root_req: + pex_warnings.warn( + "The distribution {dist} was required by the input requirement " + "{root_req} but excluded by configured excludes: " + "{excludes}".format( + dist=candidate_dist.distribution, + root_req=root_req, + excludes=excludes, + ) + ) + continue + + yield candidate_dist + for dep in iter_non_excluded_distributions( + candidate_dist.distribution.requires() + ): + yield dep + + for fingerprinted_dist in iter_non_excluded_distributions(self._requirements): + pex_builder.add_distribution( + dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint + ) + + for requirement in self._requirements: + pex_builder.add_requirement(requirement) diff --git a/pex/environment.py b/pex/environment.py index cb61b09fa..fd08c0186 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -12,6 +12,7 @@ from pex import dist_metadata, pex_warnings, targets from pex.common import pluralize from pex.dist_metadata import Distribution, Requirement +from pex.exclude_configuration import ExcludeConfiguration from pex.fingerprinted_distribution import FingerprintedDistribution from pex.inherit_path import InheritPath from pex.interpreter import PythonInterpreter @@ -345,6 +346,7 @@ def _evaluate_marker( def _resolve_requirement( self, requirement, # type: Requirement + exclude_configuration, # type: ExcludeConfiguration resolved_dists_by_key, # type: MutableMapping[_RequirementKey, FingerprintedDistribution] required, # type: bool required_by=None, # type: Optional[Distribution] @@ -354,6 +356,16 @@ def _resolve_requirement( if requirement_key in resolved_dists_by_key: return + excluded_by = exclude_configuration.excluded_by(requirement) + if excluded_by: + TRACER.log( + "Skipping resolving {requirement}: excluded by {excludes}".format( + requirement=requirement, + excludes=" and ".join(map(str, excluded_by)), + ) + ) + return + available_distributions = [ ranked_dist for ranked_dist in self._available_ranked_dists_by_project_name[ @@ -406,6 +418,7 @@ def _resolve_requirement( for not_found in self._resolve_requirement( dep_requirement, + exclude_configuration, resolved_dists_by_key, required, required_by=resolved_distribution.distribution, @@ -502,14 +515,21 @@ def resolve(self): # type: () -> Iterable[Distribution] if self._resolved_dists is None: all_reqs = [Requirement.parse(req) for req in self._pex_info.requirements] + exclude_configuration = ExcludeConfiguration.create(excluded=self._pex_info.excluded) self._resolved_dists = tuple( fingerprinted_distribution.distribution - for fingerprinted_distribution in self.resolve_dists(all_reqs) + for fingerprinted_distribution in self.resolve_dists( + all_reqs, exclude_configuration=exclude_configuration + ) ) return self._resolved_dists - def resolve_dists(self, reqs): - # type: (Iterable[Requirement]) -> Iterable[FingerprintedDistribution] + def resolve_dists( + self, + reqs, # type: Iterable[Requirement] + exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration + ): + # type: (...) -> Iterable[FingerprintedDistribution] self._update_candidate_distributions(self.iter_distributions()) @@ -536,6 +556,7 @@ def record_unresolved(dist_not_found): with TRACER.timed("Resolving {}".format(qualified_req_or_not_found.requirement), V=2): for not_found in self._resolve_requirement( requirement=qualified_req_or_not_found.requirement, + exclude_configuration=exclude_configuration, required=qualified_req_or_not_found.required, resolved_dists_by_key=resolved_dists_by_key, ): diff --git a/pex/exclude_configuration.py b/pex/exclude_configuration.py new file mode 100644 index 000000000..b4b22fc4b --- /dev/null +++ b/pex/exclude_configuration.py @@ -0,0 +1,36 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from pex.dist_metadata import Distribution, Requirement +from pex.pex_info import PexInfo +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Iterable, Tuple, Union + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s(frozen=True) +class ExcludeConfiguration(object): + @classmethod + def create(cls, excluded): + # type: (Iterable[str]) -> ExcludeConfiguration + return cls(excluded=tuple(Requirement.parse(req) for req in excluded)) + + _excluded = attr.ib(factory=tuple) # type: Tuple[Requirement, ...] + + def configure(self, pex_info): + # type: (PexInfo) -> None + for excluded in self._excluded: + pex_info.add_excluded(excluded) + + def excluded_by(self, item): + # type: (Union[Distribution, Requirement]) -> Iterable[Requirement] + if isinstance(item, Distribution): + return tuple(req for req in self._excluded if item in req) + return tuple(req for req in self._excluded if item.project_name == req.project_name) diff --git a/pex/pex_builder.py b/pex/pex_builder.py index ee70b1654..1cd69be2e 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -29,7 +29,6 @@ from pex.compiler import Compiler from pex.dist_metadata import Distribution, MetadataError from pex.enum import Enum -from pex.environment import PEXEnvironment from pex.finders import get_entry_point_from_console_script, get_script_from_distributions from pex.interpreter import PythonInterpreter from pex.layout import Layout @@ -374,21 +373,6 @@ def add_requirement(self, req): self._ensure_unfrozen("Adding a requirement") self._pex_info.add_requirement(req) - def add_from_requirements_pex(self, pex): - """Add requirements from an existing pex. - - :param pex: The path to an existing .pex file or unzipped pex directory. - """ - self._ensure_unfrozen("Adding from pex") - pex_info = PexInfo.from_pex(pex) - pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info) - for fingerprinted_dist in pex_environment.iter_distributions(): - self.add_distribution( - dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint - ) - for requirement in pex_info.requirements: - self.add_requirement(requirement) - def set_executable(self, filename, env_filename=None): """Set the executable for this environment. diff --git a/pex/pex_info.py b/pex/pex_info.py index e99700d4f..6fc9b4320 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -21,6 +21,8 @@ if TYPE_CHECKING: from typing import Any, Dict, Iterable, Mapping, Optional, Text, Tuple, Union + from pex.dist_metadata import Requirement + # N.B.: These are expensive imports and PexInfo is used during PEX bootstrapping which we want # to be as fast as possible. from pex.interpreter import PythonInterpreter @@ -144,6 +146,8 @@ def __init__(self, info=None): raise ValueError("Expected requirements to be a list, got %s" % type(requirements)) self._requirements = OrderedSet(self._parse_requirement_tuple(req) for req in requirements) + self._excluded = OrderedSet(self._pex_info.get("excluded", ())) # type: OrderedSet[str] + def _get_safe(self, key): if key not in self._pex_info: return None @@ -445,6 +449,15 @@ def add_requirement(self, requirement): def requirements(self): return self._requirements + def add_excluded(self, requirement): + # type: (Requirement) -> None + self._excluded.add(str(requirement)) + + @property + def excluded(self): + # type: () -> Iterable[str] + return self._excluded + def add_distribution(self, location, sha): self._distributions[location] = sha @@ -527,12 +540,14 @@ def update(self, other): other.interpreter_constraints ) self._requirements.update(other.requirements) + self._excluded.update(other.excluded) def as_json_dict(self): # type: () -> Dict[str, Any] data = self._pex_info.copy() data["inherit_path"] = self.inherit_path.value data["requirements"] = list(self._requirements) + data["excluded"] = list(self._excluded) data["interpreter_constraints"] = [str(ic) for ic in self.interpreter_constraints] data["distributions"] = self._distributions.copy() return data @@ -541,6 +556,7 @@ def dump(self): # type: (...) -> str data = self.as_json_dict() data["requirements"].sort() + data["excluded"].sort() data["interpreter_constraints"].sort() return json.dumps(data, sort_keys=True) diff --git a/testing/data/__init__.py b/testing/data/__init__.py new file mode 100644 index 000000000..740b3a1ff --- /dev/null +++ b/testing/data/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import pkgutil + + +def load(rel_path): + # type: (str) -> bytes + data = pkgutil.get_data(__name__, rel_path) + if data is None: + raise ValueError( + "No resource found at {rel_path} from package {name}.".format( + rel_path=rel_path, name=__name__ + ) + ) + return data + + +def path(*rel_path): + # type: (*str) -> str + path = os.path.join(os.path.dirname(__file__), *rel_path) + if not os.path.isfile(path): + raise ValueError("No resource found at {path}.".format(path=path)) + return path diff --git a/tests/data/platforms/__init__.py b/testing/data/locks/__init__.py similarity index 51% rename from tests/data/platforms/__init__.py rename to testing/data/locks/__init__.py index 87308fe2b..6227712e5 100644 --- a/tests/data/platforms/__init__.py +++ b/testing/data/locks/__init__.py @@ -1,2 +1,2 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/testing/data/locks/requests.lock.json b/testing/data/locks/requests.lock.json new file mode 100644 index 000000000..d091745b4 --- /dev/null +++ b/testing/data/locks/requests.lock.json @@ -0,0 +1,519 @@ +{ + "allow_builds": true, + "allow_prereleases": false, + "allow_wheels": true, + "build_isolation": true, + "constraints": [], + "locked_resolves": [ + { + "locked_requirements": [ + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9", + "url": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", + "url": "https://files.pythonhosted.org/packages/98/98/c2ff18671db109c9f10ed27f5ef610ae05b73bd876664139cf95bd1429aa/certifi-2023.7.22.tar.gz" + } + ], + "project_name": "certifi", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "2023.7.22" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "url": "https://files.pythonhosted.org/packages/28/76/e6222113b83e3622caa4bb41032d0b1bf785250607392e1b778aca0b8a7d/charset_normalizer-3.3.2-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "url": "https://files.pythonhosted.org/packages/05/31/e1f51c76db7be1d4aef220d29fbfa5dbb4a99165d9833dcbf166753b6dc0/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "url": "https://files.pythonhosted.org/packages/05/8c/eb854996d5fef5e4f33ad56927ad053d04dc820e4a3d39023f35cad72617/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "url": "https://files.pythonhosted.org/packages/07/07/7e554f2bbce3295e191f7e653ff15d55309a9ca40d0362fcdab36f01063c/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "url": "https://files.pythonhosted.org/packages/13/82/83c188028b6f38d39538442dd127dc794c602ae6d45d66c469f4063a4c30/charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "url": "https://files.pythonhosted.org/packages/13/f8/eefae0629fa9260f83b826ee3363e311bb03cfdd518dad1bd10d57cb2d84/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "url": "https://files.pythonhosted.org/packages/16/ea/a9e284aa38cccea06b7056d4cbc7adf37670b1f8a668a312864abf1ff7c6/charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "url": "https://files.pythonhosted.org/packages/19/28/573147271fd041d351b438a5665be8223f1dd92f273713cb882ddafe214c/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "url": "https://files.pythonhosted.org/packages/1e/49/7ab74d4ac537ece3bc3334ee08645e231f39f7d6df6347b29a74b0537103/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "url": "https://files.pythonhosted.org/packages/1f/8d/33c860a7032da5b93382cbe2873261f81467e7b37f4ed91e25fed62fd49b/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "url": "https://files.pythonhosted.org/packages/24/9d/2e3ef673dfd5be0154b20363c5cdcc5606f35666544381bee15af3778239/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "url": "https://files.pythonhosted.org/packages/2a/9d/a6d15bd1e3e2914af5955c8eb15f4071997e7078419328fee93dfd497eb7/charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "url": "https://files.pythonhosted.org/packages/2b/61/095a0aa1a84d1481998b534177c8566fdc50bb1233ea9a0478cd3cc075bd/charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "url": "https://files.pythonhosted.org/packages/2d/dc/9dacba68c9ac0ae781d40e1a0c0058e26302ea0660e574ddf6797a0347f7/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "url": "https://files.pythonhosted.org/packages/2e/37/9223632af0872c86d8b851787f0edd3fe66be4a5378f51242b25212f8374/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "url": "https://files.pythonhosted.org/packages/2e/7d/2259318c202f3d17f3fe6438149b3b9e706d1070fe3fcbb28049730bb25c/charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "url": "https://files.pythonhosted.org/packages/33/95/ef68482e4a6adf781fae8d183fb48d6f2be8facb414f49c90ba6a5149cd1/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "url": "https://files.pythonhosted.org/packages/33/c3/3b96a435c5109dd5b6adc8a59ba1d678b302a97938f032e3770cc84cd354/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "url": "https://files.pythonhosted.org/packages/34/2a/f392457d45e24a0c9bfc012887ed4f3c54bf5d4d05a5deb970ffec4b7fc0/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "url": "https://files.pythonhosted.org/packages/3a/52/9f9d17c3b54dc238de384c4cb5a2ef0e27985b42a0e5cc8e8a31d918d48d/charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "url": "https://files.pythonhosted.org/packages/3d/09/d82fe4a34c5f0585f9ea1df090e2a71eb9bb1e469723053e1ee9f57c16f3/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "url": "https://files.pythonhosted.org/packages/3d/85/5b7416b349609d20611a64718bed383b9251b5a601044550f0c8983b8900/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "url": "https://files.pythonhosted.org/packages/3e/33/21a875a61057165e92227466e54ee076b73af1e21fe1b31f1e292251aa1e/charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "url": "https://files.pythonhosted.org/packages/3f/ba/3f5e7be00b215fa10e13d64b1f6237eb6ebea66676a41b2bcdd09fe74323/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "url": "https://files.pythonhosted.org/packages/40/26/f35951c45070edc957ba40a5b1db3cf60a9dbb1b350c2d5bef03e01e61de/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "url": "https://files.pythonhosted.org/packages/43/05/3bf613e719efe68fb3a77f9c536a389f35b95d75424b96b426a47a45ef1d/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "url": "https://files.pythonhosted.org/packages/44/80/b339237b4ce635b4af1c73742459eee5f97201bd92b2371c53e11958392e/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "url": "https://files.pythonhosted.org/packages/45/59/3d27019d3b447a88fe7e7d004a1e04be220227760264cc41b405e863891b/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "url": "https://files.pythonhosted.org/packages/46/6a/d5c26c41c49b546860cc1acabdddf48b0b3fb2685f4f5617ac59261b44ae/charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "url": "https://files.pythonhosted.org/packages/4f/d1/d547cc26acdb0cc458b152f79b2679d7422f29d41581e6fa907861e88af1/charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "url": "https://files.pythonhosted.org/packages/51/fd/0ee5b1c2860bb3c60236d05b6e4ac240cf702b67471138571dad91bcfed8/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "url": "https://files.pythonhosted.org/packages/53/cd/aa4b8a4d82eeceb872f83237b2d27e43e637cac9ffaef19a1321c3bafb67/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561", + "url": "https://files.pythonhosted.org/packages/54/7f/cad0b328759630814fcf9d804bfabaf47776816ad4ef2e9938b7e1123d04/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "url": "https://files.pythonhosted.org/packages/58/78/a0bc646900994df12e07b4ae5c713f2b3e5998f58b9d3720cce2aa45652f/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "url": "https://files.pythonhosted.org/packages/58/a2/0c63d5d7ffac3104b86631b7f2690058c97bf72d3145c0a9cd4fb90c58c2/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "url": "https://files.pythonhosted.org/packages/5b/ae/ce2c12fcac59cb3860b2e2d76dc405253a4475436b1861d95fe75bdea520/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "url": "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "url": "https://files.pythonhosted.org/packages/66/fe/c7d3da40a66a6bf2920cce0f436fa1f62ee28aaf92f412f0bf3b84c8ad6c/charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "url": "https://files.pythonhosted.org/packages/68/77/02839016f6fbbf808e8b38601df6e0e66c17bbab76dff4613f7511413597/charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "url": "https://files.pythonhosted.org/packages/72/1a/641d5c9f59e6af4c7b53da463d07600a695b9824e20849cb6eea8a627761/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "url": "https://files.pythonhosted.org/packages/74/f1/0d9fe69ac441467b737ba7f48c68241487df2f4522dd7246d9426e7c690e/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "url": "https://files.pythonhosted.org/packages/79/66/8946baa705c588521afe10b2d7967300e49380ded089a62d38537264aece/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "url": "https://files.pythonhosted.org/packages/7b/ef/5eb105530b4da8ae37d506ccfa25057961b7b63d581def6f99165ea89c7e/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "url": "https://files.pythonhosted.org/packages/81/b2/160893421adfa3c45554fb418e321ed342bb10c0a4549e855b2b2a3699cb/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "url": "https://files.pythonhosted.org/packages/8d/b7/9e95102e9a8cce6654b85770794b582dda2921ec1fd924c10fbcf215ad31/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "url": "https://files.pythonhosted.org/packages/91/33/749df346e93d7a30cdcb90cbfdd41a06026317bfbfb62cd68307c1a3c543/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "url": "https://files.pythonhosted.org/packages/91/95/e2cfa7ce962e6c4b59a44a6e19e541c3a0317e543f0e0923f844e8d7d21d/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "url": "https://files.pythonhosted.org/packages/98/69/5d8751b4b670d623aa7a47bef061d69c279e9f922f6705147983aa76c3ce/charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "url": "https://files.pythonhosted.org/packages/99/b0/9c365f6d79a9f0f3c379ddb40a256a67aa69c59609608fe7feb6235896e1/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "url": "https://files.pythonhosted.org/packages/9e/ef/cd47a63d3200b232792e361cd67530173a09eb011813478b1c0fb8aa7226/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "url": "https://files.pythonhosted.org/packages/a0/b1/4e72ef73d68ebdd4748f2df97130e8428c4625785f2b6ece31f555590c2d/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "url": "https://files.pythonhosted.org/packages/a2/51/e5023f937d7f307c948ed3e5c29c4b7a3e42ed2ee0b8cdf8f3a706089bf0/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "url": "https://files.pythonhosted.org/packages/a8/31/47d018ef89f95b8aded95c589a77c072c55e94b50a41aa99c0a2008a45a4/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "url": "https://files.pythonhosted.org/packages/a8/6f/4ff299b97da2ed6358154b6eb3a2db67da2ae204e53d205aacb18a7e4f34/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "url": "https://files.pythonhosted.org/packages/b2/62/5a5dcb9a71390a9511a253bde19c9c89e0b20118e41080185ea69fb2c209/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "url": "https://files.pythonhosted.org/packages/b3/c1/ebca8e87c714a6a561cfee063f0655f742e54b8ae6e78151f60ba8708b3a/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "url": "https://files.pythonhosted.org/packages/b8/60/e2f67915a51be59d4539ed189eb0a2b0d292bf79270410746becb32bc2c3/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "url": "https://files.pythonhosted.org/packages/bd/28/7ea29e73eea52c7e15b4b9108d0743fc9e4cc2cdb00d275af1df3d46d360/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "url": "https://files.pythonhosted.org/packages/be/4d/9e370f8281cec2fcc9452c4d1ac513324c32957c5f70c73dd2fa8442a21a/charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "url": "https://files.pythonhosted.org/packages/c2/65/52aaf47b3dd616c11a19b1052ce7fa6321250a7a0b975f48d8c366733b9f/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "url": "https://files.pythonhosted.org/packages/c9/7a/6d8767fac16f2c80c7fa9f14e0f53d4638271635c306921844dc0b5fd8a6/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "url": "https://files.pythonhosted.org/packages/cc/94/f7cf5e5134175de79ad2059edf2adce18e0685ebdb9227ff0139975d0e93/charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "url": "https://files.pythonhosted.org/packages/cf/7c/f3b682fa053cc21373c9a839e6beba7705857075686a05c72e0f8c4980ca/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "url": "https://files.pythonhosted.org/packages/d1/2f/0d1efd07c74c52b6886c32a3b906fb8afd2fecf448650e73ecb90a5a27f1/charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "url": "https://files.pythonhosted.org/packages/d1/b2/fcedc8255ec42afee97f9e6f0145c734bbe104aac28300214593eb326f1d/charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "url": "https://files.pythonhosted.org/packages/d8/b5/eb705c313100defa57da79277d9207dc8d8e45931035862fa64b625bfead/charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "url": "https://files.pythonhosted.org/packages/da/f1/3702ba2a7470666a62fd81c58a4c40be00670e5006a67f4d626e57f013ae/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "url": "https://files.pythonhosted.org/packages/dd/51/68b61b90b24ca35495956b718f35a9756ef7d3dd4b3c1508056fa98d1a1b/charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "url": "https://files.pythonhosted.org/packages/df/3e/a06b18788ca2eb6695c9b22325b6fde7dde0f1d1838b1792a0076f58fe9d/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "url": "https://files.pythonhosted.org/packages/e1/9c/60729bf15dc82e3aaf5f71e81686e42e50715a1399770bcde1a9e43d09db/charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "url": "https://files.pythonhosted.org/packages/e4/a6/7ee57823d46331ddc37dd00749c95b0edec2c79b15fc0d6e6efb532e89ac/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "url": "https://files.pythonhosted.org/packages/eb/5c/97d97248af4920bc68687d9c3b3c0f47c910e21a8ff80af4565a576bd2f0/charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "url": "https://files.pythonhosted.org/packages/ee/fb/14d30eb4956408ee3ae09ad34299131fb383c47df355ddb428a7331cfa1e/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "url": "https://files.pythonhosted.org/packages/ef/d4/a1d72a8f6aa754fdebe91b848912025d30ab7dced61e9ed8aabbf791ed65/charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "url": "https://files.pythonhosted.org/packages/f2/0e/e06bc07ef4673e4d24dc461333c254586bb759fdd075031539bab6514d07/charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "url": "https://files.pythonhosted.org/packages/f6/93/bb6cbeec3bf9da9b2eba458c15966658d1daa8b982c642f81c93ad9b40e1/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "url": "https://files.pythonhosted.org/packages/f6/d3/bfc699ab2c4f9245867060744e8136d359412ff1e5ad93be38a46d160f9d/charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "url": "https://files.pythonhosted.org/packages/f7/9d/bcf4a449a438ed6f19790eee543a86a740c77508fbc5ddab210ab3ba3a9a/charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl" + } + ], + "project_name": "charset-normalizer", + "requires_dists": [], + "requires_python": ">=3.7.0", + "version": "3.3.2" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2", + "url": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "url": "https://files.pythonhosted.org/packages/8b/e1/43beb3d38dba6cb420cefa297822eac205a277ab43e5ba5d5c46faf96438/idna-3.4.tar.gz" + } + ], + "project_name": "idna", + "requires_dists": [], + "requires_python": ">=3.5", + "version": "3.4" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", + "url": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1", + "url": "https://files.pythonhosted.org/packages/9d/be/10918a2eac4ae9f02f6cfe6414b7a155ccd8f7f9d4380d62fd5b955065c3/requests-2.31.0.tar.gz" + } + ], + "project_name": "requests", + "requires_dists": [ + "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"", + "certifi>=2017.4.17", + "chardet<6,>=3.0.2; extra == \"use-chardet-on-py3\"", + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1" + ], + "requires_python": ">=3.7", + "version": "2.31.0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e", + "url": "https://files.pythonhosted.org/packages/d2/b2/b157855192a68541a91ba7b2bbcb91f1b4faa51f8bae38d8005c034be524/urllib3-2.0.7-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", + "url": "https://files.pythonhosted.org/packages/af/47/b215df9f71b4fdba1025fc05a77db2ad243fa0926755a52c5e71659f4e3c/urllib3-2.0.7.tar.gz" + } + ], + "project_name": "urllib3", + "requires_dists": [ + "brotli>=1.0.9; platform_python_implementation == \"CPython\" and extra == \"brotli\"", + "brotlicffi>=0.8.0; platform_python_implementation != \"CPython\" and extra == \"brotli\"", + "certifi; extra == \"secure\"", + "cryptography>=1.9; extra == \"secure\"", + "idna>=2.0.0; extra == \"secure\"", + "pyopenssl>=17.1.0; extra == \"secure\"", + "pysocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"", + "urllib3-secure-extra; extra == \"secure\"", + "zstandard>=0.18.0; extra == \"zstd\"" + ], + "requires_python": ">=3.7", + "version": "2.0.7" + } + ], + "platform_tag": null + } + ], + "path_mappings": {}, + "pex_version": "2.1.150", + "pip_version": "23.2", + "prefer_older_binary": false, + "requirements": [ + "requests" + ], + "requires_python": [ + "<3.13,>=3.7" + ], + "resolver_version": "pip-2020-resolver", + "style": "universal", + "target_systems": [ + "linux", + "mac" + ], + "transitive": true, + "use_pep517": null +} diff --git a/tests/data/__init__.py b/testing/data/platforms/__init__.py similarity index 100% rename from tests/data/__init__.py rename to testing/data/platforms/__init__.py diff --git a/tests/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt b/testing/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt similarity index 100% rename from tests/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt rename to testing/data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt diff --git a/tests/integration/test_excludes.py b/tests/integration/test_excludes.py new file mode 100644 index 000000000..9896b51df --- /dev/null +++ b/tests/integration/test_excludes.py @@ -0,0 +1,144 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import os.path +import subprocess +from os.path import commonprefix + +import pytest + +from pex.executor import Executor +from pex.pep_503 import ProjectName +from pex.pex import PEX +from pex.pex_info import PexInfo +from pex.typing import TYPE_CHECKING +from pex.venv.virtualenv import Virtualenv +from testing import PY_VER, data, make_env, run_pex_command + +if TYPE_CHECKING: + from typing import Any, Tuple + + +@pytest.fixture(scope="module") +def requests_certifi_excluded_pex(tmpdir_factory): + # type: (Any) -> str + + requests_lock = data.path("locks", "requests.lock.json") + pex_root = str(tmpdir_factory.mktemp("pex_root")) + pex = str(tmpdir_factory.mktemp("pex")) + run_pex_command( + args=[ + "--lock", + requests_lock, + "--exclude", + "certifi", + "-o", + pex, + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + ] + ).assert_success() + + assert ProjectName("certifi") not in frozenset( + dist.metadata.project_name for dist in PEX(pex).resolve() + ) + + return pex + + +REQUESTS_CMD = ["-c", "import requests, sys; print(sys.modules['certifi'].__file__)"] +EXPECTED_IMPORT_ERROR_MSG = "ModuleNotFoundError: No module named 'certifi'" + + +@pytest.fixture(scope="module") +def certifi_venv(tmpdir_factory): + # type: (Any) -> Virtualenv + + venv = Virtualenv.create(venv_dir=str(tmpdir_factory.mktemp("venv"))) + pip = venv.install_pip() + + # N.B.: The constraining lock requirement is the one expressed by requests: certifi>=2017.4.17 + # The actual locked version is 2023.7.22; so we stress this crease and use a different, but + # allowed, version. + subprocess.check_call(args=[pip, "install", "certifi==2017.4.17"]) + + return venv + + +skip_unless_37_to_312 = pytest.mark.skipif( + PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13" +) + + +def assert_certifi_import_behavior( + pex, # type: str + certifi_venv, # type: Virtualenv +): + requests_cmd = [pex] + REQUESTS_CMD + + # Although the venv has certifi available, a PEX is hermetic by default; so it shouldn't be + # used. + with pytest.raises(Executor.NonZeroExit) as exc: + certifi_venv.interpreter.execute(args=requests_cmd) + assert EXPECTED_IMPORT_ERROR_MSG in exc.value.stderr + + # Allowing the `sys.path` to be inherited should allow the certifi hole to be filled in. + _, stdout, _ = certifi_venv.interpreter.execute( + args=requests_cmd, env=make_env(PEX_INHERIT_PATH="fallback") + ) + assert certifi_venv.site_packages_dir == commonprefix( + [certifi_venv.site_packages_dir, stdout.strip()] + ) + + +@skip_unless_37_to_312 +def test_exclude( + tmpdir, # type: Any + requests_certifi_excluded_pex, # type: str + certifi_venv, # type: Virtualenv +): + # type: (...) -> None + + requests_cmd = [requests_certifi_excluded_pex] + REQUESTS_CMD + + # The exclude option is buyer beware. A PEX using this option will not work if the excluded + # distributions carry modules that are, in fact, needed at run time. + process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = process.communicate() + assert process.returncode != 0 + assert EXPECTED_IMPORT_ERROR_MSG in stderr.decode("utf-8"), stderr.decode("utf-8") + + assert_certifi_import_behavior(requests_certifi_excluded_pex, certifi_venv) + + +@skip_unless_37_to_312 +def test_requirements_pex_exclude( + tmpdir, # type: Any + requests_certifi_excluded_pex, # type: str + certifi_venv, # type: Virtualenv +): + # type: (...) -> None + + pex_root = PexInfo.from_pex(requests_certifi_excluded_pex).pex_root + pex = os.path.join(str(tmpdir), "pex") + run_pex_command( + args=[ + "--requirements-pex", + requests_certifi_excluded_pex, + "ansicolors==1.1.8", + "-o", + pex, + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + ] + ).assert_success() + + # Shouldn't need the certifi hole filled to import colors. + output = subprocess.check_output(args=[pex, "-c", "import colors; print(colors.__file__)"]) + assert pex_root == commonprefix([pex_root, output.decode("utf-8").strip()]) + + assert_certifi_import_behavior(pex, certifi_venv) diff --git a/tests/test_dependency_manager.py b/tests/test_dependency_manager.py new file mode 100644 index 000000000..98648f9e4 --- /dev/null +++ b/tests/test_dependency_manager.py @@ -0,0 +1,155 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import hashlib +import os.path +import warnings + +import pytest + +from pex.dependency_manager import DependencyManager +from pex.dist_metadata import DistMetadata, Distribution, Requirement +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.orderedset import OrderedSet +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pex.pex_warnings import PEXWarning +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Tuple + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s(frozen=True) +class DistFactory(object): + install_base_dir = attr.ib() # type: str + + def create( + self, + name, # type: str + *requires # type: str + ): + # type: (...) -> FingerprintedDistribution + fingerprint = hashlib.sha256(name.encode("utf-8")).hexdigest() + location = os.path.join(self.install_base_dir, fingerprint, name) + os.makedirs(location) + return FingerprintedDistribution( + distribution=Distribution( + location=location, + metadata=DistMetadata( + project_name=ProjectName(name), + version=Version("0.1.0"), + requires_dists=tuple(Requirement.parse(req) for req in requires), + ), + ), + fingerprint=fingerprint, + ) + + +@pytest.fixture +def dist_factory(tmpdir): + # type: (Any) -> DistFactory + return DistFactory(os.path.join(str(tmpdir), "installed_wheels")) + + +@attr.s(frozen=True) +class DistGraph(object): + root_reqs = attr.ib() # type: Tuple[Requirement, ...] + dists = attr.ib() # type: Tuple[FingerprintedDistribution, ...] + + def dist(self, name): + # type: (str) -> FingerprintedDistribution + project_name = ProjectName(name) + dists = [ + dist for dist in self.dists if project_name == dist.distribution.metadata.project_name + ] + assert len(dists) == 1, "Expected {name} to match one dist, found {found}".format( + name=name, found=" ".join(map(str, dists)) if dists else "none" + ) + return dists[0] + + +@pytest.fixture +def dist_graph(dist_factory): + # type: (DistFactory) -> DistGraph + + # distA distB <--------\ + # \ / \ | + # v v v | + # distC distD (cycle) + # / \ / | + # V v v | + # distE distF ---------/ + + return DistGraph( + root_reqs=(Requirement.parse("a"), Requirement.parse("b")), + dists=( + dist_factory.create("A", "c"), + dist_factory.create("B", "c", "d"), + dist_factory.create("C", "e", "f"), + dist_factory.create("D", "f"), + dist_factory.create("E"), + dist_factory.create("F", "b"), + ), + ) + + +def test_exclude_root_reqs(dist_graph): + # type: (DistGraph) -> None + + dependency_manager = DependencyManager( + requirements=OrderedSet(dist_graph.root_reqs), distributions=OrderedSet(dist_graph.dists) + ) + + pex_info = PexInfo.default() + pex_builder = PEXBuilder(pex_info=pex_info) + + with warnings.catch_warnings(record=True) as events: + dependency_manager.configure(pex_builder, excluded=["a", "b"]) + assert 2 == len(events) + + warning = events[0] + assert PEXWarning == warning.category + assert ( + "The distribution A 0.1.0 was required by the input requirement a but excluded by " + "configured excludes: a" + ) == str(warning.message) + + warning = events[1] + assert PEXWarning == warning.category + assert ( + "The distribution B 0.1.0 was required by the input requirement b but excluded by " + "configured excludes: b" + ) == str(warning.message) + + pex_builder.freeze() + + assert ["a", "b"] == list(pex_info.requirements) + assert ["a", "b"] == list(pex_info.excluded) + assert {} == pex_info.distributions + + +def test_exclude_complex(dist_graph): + # type: (DistGraph) -> None + + dependency_manager = DependencyManager( + requirements=OrderedSet(dist_graph.root_reqs), distributions=OrderedSet(dist_graph.dists) + ) + + pex_info = PexInfo.default() + pex_builder = PEXBuilder(pex_info=pex_info) + dependency_manager.configure(pex_builder, excluded=["c"]) + pex_builder.freeze() + + assert ["a", "b"] == list(pex_info.requirements) + assert ["c"] == list(pex_info.excluded) + expected_dists = [dist_graph.dist(name) for name in ("A", "B", "D", "F")] + assert { + os.path.basename(dist.location): dist.fingerprint for dist in expected_dists + } == pex_info.distributions diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index cafaf2ebb..e99abe05b 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -210,43 +210,6 @@ def test_pex_builder_deterministic_timestamp(): assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist()) -def test_pex_builder_from_requirements_pex(): - # type: () -> None - def build_from_req_pex(path, req_pex): - # type: (str, str) -> PEXBuilder - pb = PEXBuilder(path=path) - pb.add_from_requirements_pex(req_pex) - with open(os.path.join(path, "exe.py"), "w") as fp: - fp.write(exe_main) - pb.set_executable(os.path.join(path, "exe.py")) - pb.freeze() - return pb - - def verify(pb): - # type: (PEXBuilder) -> None - success_txt = os.path.join(pb.path(), "success.txt") - PEX(pb.path(), interpreter=pb.interpreter).run(args=[success_txt]) - assert os.path.exists(success_txt) - with open(success_txt) as fp: - assert fp.read() == "success" - - # Build from pex dir. - with temporary_dir() as td2: - with temporary_dir() as td1, make_bdist("p1") as p1: - pb1 = write_pex(td1, dists=[p1]) - pb2 = build_from_req_pex(td2, pb1.path()) - verify(pb2) - - # Build from .pex file. - with temporary_dir() as td4: - with temporary_dir() as td3, make_bdist("p1") as p1: - pb3 = write_pex(td3, dists=[p1]) - target = os.path.join(td3, "foo.pex") - pb3.build(target) - pb4 = build_from_req_pex(td4, target) - verify(pb4) - - def test_pex_builder_script_from_pex_path(tmpdir): # type: (Any) -> None diff --git a/tests/test_platform.py b/tests/test_platform.py index 49943758d..178f93249 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -11,6 +11,7 @@ from pex.pep_425 import CompatibilityTags from pex.platforms import Platform from pex.third_party.packaging import tags +from testing import data EXPECTED_BASE = [("py27", "none", "any"), ("py2", "none", "any")] @@ -130,7 +131,7 @@ def test_platform_supported_tags(): # A golden file test. This could break if we upgrade Pip and it upgrades packaging which, from # time to time, corrects omissions in tag sets. - golden_tags = pkgutil.get_data(__name__, "data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt") + golden_tags = data.load("platforms/macosx_10_13_x86_64-cp-36-m.tags.txt") assert golden_tags is not None assert ( CompatibilityTags(