diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 884f7b305..1d9bab0dc 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -826,11 +826,14 @@ def build_pex( pex_info.interpreter_constraints = interpreter_constraints dependency_manager = DependencyManager() + excluded = list(options.excluded) # type: List[str] + with TRACER.timed( "Adding distributions from pexes: {}".format(" ".join(options.requirements_pexes)) ): for requirements_pex in options.requirements_pexes: - dependency_manager.add_from_pex(requirements_pex) + requirements_pex_info = dependency_manager.add_from_pex(requirements_pex) + excluded.extend(requirements_pex_info.excluded) with TRACER.timed( "Resolving distributions for requirements: {}".format( @@ -858,7 +861,7 @@ def build_pex( die(str(e)) with TRACER.timed("Configuring PEX dependencies"): - dependency_manager.configure(pex_builder, excluded=options.excluded) + dependency_manager.configure(pex_builder, excluded=excluded) if options.entry_point: pex_builder.set_entry_point(options.entry_point) diff --git a/pex/dependency_manager.py b/pex/dependency_manager.py index 8a2557173..1cce6523d 100644 --- a/pex/dependency_manager.py +++ b/pex/dependency_manager.py @@ -32,7 +32,7 @@ class DependencyManager(object): _distributions = attr.ib(factory=OrderedSet) # type: OrderedSet[FingerprintedDistribution] def add_from_pex(self, pex): - # type: (str) -> None + # type: (str) -> PexInfo pex_info = PexInfo.from_pex(pex) self._requirements.update(Requirement.parse(req) for req in pex_info.requirements) @@ -40,6 +40,8 @@ def add_from_pex(self, pex): pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info) self._distributions.update(pex_environment.iter_distributions()) + return pex_info + def add_from_installed(self, installed): # type: (Installed) -> None @@ -63,6 +65,12 @@ def configure( for dist in self._distributions: dists_by_project_name[dist.distribution.metadata.project_name].add(dist) + root_requirements_by_project_name = defaultdict( + OrderedSet + ) # type: DefaultDict[ProjectName, OrderedSet[Requirement]] + for root_req in self._requirements: + root_requirements_by_project_name[root_req.project_name].add(root_req) + def iter_non_excluded_distributions(requirements): # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution] for req in requirements: @@ -80,7 +88,9 @@ def iter_non_excluded_distributions(requirements): candidate=candidate_dist.distribution, excludes=excludes ) ) - for root_req in self._requirements: + for root_req in root_requirements_by_project_name[ + candidate_dist.distribution.metadata.project_name + ]: if candidate_dist.distribution in root_req: pex_warnings.warn( "The distribution {dist} was required by the input requirement " diff --git a/pex/environment.py b/pex/environment.py index 6cc26ce0f..fd08c0186 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -346,10 +346,10 @@ def _evaluate_marker( def _resolve_requirement( self, requirement, # type: Requirement + exclude_configuration, # type: ExcludeConfiguration resolved_dists_by_key, # type: MutableMapping[_RequirementKey, FingerprintedDistribution] required, # type: bool required_by=None, # type: Optional[Distribution] - exclude_configuration=ExcludeConfiguration(), # type: ExcludeConfiguration ): # type: (...) -> Iterator[_DistributionNotFound] requirement_key = _RequirementKey.create(requirement) @@ -418,10 +418,10 @@ def _resolve_requirement( for not_found in self._resolve_requirement( dep_requirement, + exclude_configuration, resolved_dists_by_key, required, required_by=resolved_distribution.distribution, - exclude_configuration=exclude_configuration, ): yield not_found @@ -556,9 +556,9 @@ def record_unresolved(dist_not_found): with TRACER.timed("Resolving {}".format(qualified_req_or_not_found.requirement), V=2): for not_found in self._resolve_requirement( requirement=qualified_req_or_not_found.requirement, + exclude_configuration=exclude_configuration, required=qualified_req_or_not_found.required, resolved_dists_by_key=resolved_dists_by_key, - exclude_configuration=exclude_configuration, ): record_unresolved(not_found) diff --git a/pex/pex_builder.py b/pex/pex_builder.py index ee70b1654..1cd69be2e 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -29,7 +29,6 @@ from pex.compiler import Compiler from pex.dist_metadata import Distribution, MetadataError from pex.enum import Enum -from pex.environment import PEXEnvironment from pex.finders import get_entry_point_from_console_script, get_script_from_distributions from pex.interpreter import PythonInterpreter from pex.layout import Layout @@ -374,21 +373,6 @@ def add_requirement(self, req): self._ensure_unfrozen("Adding a requirement") self._pex_info.add_requirement(req) - def add_from_requirements_pex(self, pex): - """Add requirements from an existing pex. - - :param pex: The path to an existing .pex file or unzipped pex directory. - """ - self._ensure_unfrozen("Adding from pex") - pex_info = PexInfo.from_pex(pex) - pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info) - for fingerprinted_dist in pex_environment.iter_distributions(): - self.add_distribution( - dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint - ) - for requirement in pex_info.requirements: - self.add_requirement(requirement) - def set_executable(self, filename, env_filename=None): """Set the executable for this environment. diff --git a/tests/integration/test_excludes.py b/tests/integration/test_excludes.py index 41d520014..9896b51df 100644 --- a/tests/integration/test_excludes.py +++ b/tests/integration/test_excludes.py @@ -10,21 +10,22 @@ from pex.executor import Executor from pex.pep_503 import ProjectName from pex.pex import PEX +from pex.pex_info import PexInfo from pex.typing import TYPE_CHECKING from pex.venv.virtualenv import Virtualenv from testing import PY_VER, data, make_env, run_pex_command if TYPE_CHECKING: - from typing import Any + from typing import Any, Tuple -@pytest.mark.skipif(PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13") -def test_exclude(tmpdir): - # type: (Any) -> None +@pytest.fixture(scope="module") +def requests_certifi_excluded_pex(tmpdir_factory): + # type: (Any) -> str requests_lock = data.path("locks", "requests.lock.json") - pex_root = os.path.join(str(tmpdir), "pex_root") - pex = os.path.join(str(tmpdir), "pex") + pex_root = str(tmpdir_factory.mktemp("pex_root")) + pex = str(tmpdir_factory.mktemp("pex")) run_pex_command( args=[ "--lock", @@ -44,19 +45,18 @@ def test_exclude(tmpdir): dist.metadata.project_name for dist in PEX(pex).resolve() ) - # The exclude option is buyer beware. A PEX using this option will not work if the excluded - # distributions carry modules that are, in fact, needed at run time. - requests_cmd = [pex, "-c", "import requests, sys; print(sys.modules['certifi'].__file__)"] - expected_import_error_msg = "ModuleNotFoundError: No module named 'certifi'" + return pex + + +REQUESTS_CMD = ["-c", "import requests, sys; print(sys.modules['certifi'].__file__)"] +EXPECTED_IMPORT_ERROR_MSG = "ModuleNotFoundError: No module named 'certifi'" - process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _, stderr = process.communicate() - assert process.returncode != 0 - assert expected_import_error_msg in stderr.decode("utf-8"), stderr.decode("utf-8") +@pytest.fixture(scope="module") +def certifi_venv(tmpdir_factory): + # type: (Any) -> Virtualenv - venv_dir = os.path.join(str(tmpdir), "venv") - venv = Virtualenv.create(venv_dir) + venv = Virtualenv.create(venv_dir=str(tmpdir_factory.mktemp("venv"))) pip = venv.install_pip() # N.B.: The constraining lock requirement is the one expressed by requests: certifi>=2017.4.17 @@ -64,14 +64,81 @@ def test_exclude(tmpdir): # allowed, version. subprocess.check_call(args=[pip, "install", "certifi==2017.4.17"]) + return venv + + +skip_unless_37_to_312 = pytest.mark.skipif( + PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13" +) + + +def assert_certifi_import_behavior( + pex, # type: str + certifi_venv, # type: Virtualenv +): + requests_cmd = [pex] + REQUESTS_CMD + # Although the venv has certifi available, a PEX is hermetic by default; so it shouldn't be # used. with pytest.raises(Executor.NonZeroExit) as exc: - venv.interpreter.execute(args=requests_cmd) - assert expected_import_error_msg in exc.value.stderr + certifi_venv.interpreter.execute(args=requests_cmd) + assert EXPECTED_IMPORT_ERROR_MSG in exc.value.stderr # Allowing the `sys.path` to be inherited should allow the certifi hole to be filled in. - _, stdout, _ = venv.interpreter.execute( + _, stdout, _ = certifi_venv.interpreter.execute( args=requests_cmd, env=make_env(PEX_INHERIT_PATH="fallback") ) - assert venv.site_packages_dir == commonprefix([venv.site_packages_dir, stdout.strip()]) + assert certifi_venv.site_packages_dir == commonprefix( + [certifi_venv.site_packages_dir, stdout.strip()] + ) + + +@skip_unless_37_to_312 +def test_exclude( + tmpdir, # type: Any + requests_certifi_excluded_pex, # type: str + certifi_venv, # type: Virtualenv +): + # type: (...) -> None + + requests_cmd = [requests_certifi_excluded_pex] + REQUESTS_CMD + + # The exclude option is buyer beware. A PEX using this option will not work if the excluded + # distributions carry modules that are, in fact, needed at run time. + process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = process.communicate() + assert process.returncode != 0 + assert EXPECTED_IMPORT_ERROR_MSG in stderr.decode("utf-8"), stderr.decode("utf-8") + + assert_certifi_import_behavior(requests_certifi_excluded_pex, certifi_venv) + + +@skip_unless_37_to_312 +def test_requirements_pex_exclude( + tmpdir, # type: Any + requests_certifi_excluded_pex, # type: str + certifi_venv, # type: Virtualenv +): + # type: (...) -> None + + pex_root = PexInfo.from_pex(requests_certifi_excluded_pex).pex_root + pex = os.path.join(str(tmpdir), "pex") + run_pex_command( + args=[ + "--requirements-pex", + requests_certifi_excluded_pex, + "ansicolors==1.1.8", + "-o", + pex, + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + ] + ).assert_success() + + # Shouldn't need the certifi hole filled to import colors. + output = subprocess.check_output(args=[pex, "-c", "import colors; print(colors.__file__)"]) + assert pex_root == commonprefix([pex_root, output.decode("utf-8").strip()]) + + assert_certifi_import_behavior(pex, certifi_venv) diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index cafaf2ebb..e99abe05b 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -210,43 +210,6 @@ def test_pex_builder_deterministic_timestamp(): assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist()) -def test_pex_builder_from_requirements_pex(): - # type: () -> None - def build_from_req_pex(path, req_pex): - # type: (str, str) -> PEXBuilder - pb = PEXBuilder(path=path) - pb.add_from_requirements_pex(req_pex) - with open(os.path.join(path, "exe.py"), "w") as fp: - fp.write(exe_main) - pb.set_executable(os.path.join(path, "exe.py")) - pb.freeze() - return pb - - def verify(pb): - # type: (PEXBuilder) -> None - success_txt = os.path.join(pb.path(), "success.txt") - PEX(pb.path(), interpreter=pb.interpreter).run(args=[success_txt]) - assert os.path.exists(success_txt) - with open(success_txt) as fp: - assert fp.read() == "success" - - # Build from pex dir. - with temporary_dir() as td2: - with temporary_dir() as td1, make_bdist("p1") as p1: - pb1 = write_pex(td1, dists=[p1]) - pb2 = build_from_req_pex(td2, pb1.path()) - verify(pb2) - - # Build from .pex file. - with temporary_dir() as td4: - with temporary_dir() as td3, make_bdist("p1") as p1: - pb3 = write_pex(td3, dists=[p1]) - target = os.path.join(td3, "foo.pex") - pb3.build(target) - pb4 = build_from_req_pex(td4, target) - verify(pb4) - - def test_pex_builder_script_from_pex_path(tmpdir): # type: (Any) -> None