Various fixes suggested by Huon.

pex/bin/pex.py

@@ -826,11 +826,14 @@ def build_pex(
     pex_info.interpreter_constraints = interpreter_constraints
 
     dependency_manager = DependencyManager()
+    excluded = list(options.excluded)  # type: List[str]
+
     with TRACER.timed(
         "Adding distributions from pexes: {}".format(" ".join(options.requirements_pexes))
     ):
         for requirements_pex in options.requirements_pexes:
-            dependency_manager.add_from_pex(requirements_pex)
+            requirements_pex_info = dependency_manager.add_from_pex(requirements_pex)
+            excluded.extend(requirements_pex_info.excluded)
 
     with TRACER.timed(
         "Resolving distributions for requirements: {}".format(
@@ -858,7 +861,7 @@ def build_pex(
             die(str(e))
 
     with TRACER.timed("Configuring PEX dependencies"):
-        dependency_manager.configure(pex_builder, excluded=options.excluded)
+        dependency_manager.configure(pex_builder, excluded=excluded)
 
     if options.entry_point:
         pex_builder.set_entry_point(options.entry_point)

pex/dependency_manager.py

@@ -32,14 +32,16 @@ class DependencyManager(object):
     _distributions = attr.ib(factory=OrderedSet)  # type: OrderedSet[FingerprintedDistribution]
 
     def add_from_pex(self, pex):
-        # type: (str) -> None
+        # type: (str) -> PexInfo
 
         pex_info = PexInfo.from_pex(pex)
         self._requirements.update(Requirement.parse(req) for req in pex_info.requirements)
 
         pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info)
         self._distributions.update(pex_environment.iter_distributions())
 
+        return pex_info
+
     def add_from_installed(self, installed):
         # type: (Installed) -> None
 
@@ -63,6 +65,12 @@ def configure(
         for dist in self._distributions:
             dists_by_project_name[dist.distribution.metadata.project_name].add(dist)
 
+        root_requirements_by_project_name = defaultdict(
+            OrderedSet
+        )  # type: DefaultDict[ProjectName, OrderedSet[Requirement]]
+        for root_req in self._requirements:
+            root_requirements_by_project_name[root_req.project_name].add(root_req)
+
         def iter_non_excluded_distributions(requirements):
             # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution]
             for req in requirements:
@@ -80,7 +88,9 @@ def iter_non_excluded_distributions(requirements):
                                 candidate=candidate_dist.distribution, excludes=excludes
                             )
                         )
-                        for root_req in self._requirements:
+                        for root_req in root_requirements_by_project_name[
+                            candidate_dist.distribution.metadata.project_name
+                        ]:
                             if candidate_dist.distribution in root_req:
                                 pex_warnings.warn(
                                     "The distribution {dist} was required by the input requirement "

pex/environment.py

@@ -346,10 +346,10 @@ def _evaluate_marker(
     def _resolve_requirement(
         self,
         requirement,  # type: Requirement
+        exclude_configuration,  # type: ExcludeConfiguration
         resolved_dists_by_key,  # type: MutableMapping[_RequirementKey, FingerprintedDistribution]
         required,  # type: bool
         required_by=None,  # type: Optional[Distribution]
-        exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
     ):
         # type: (...) -> Iterator[_DistributionNotFound]
         requirement_key = _RequirementKey.create(requirement)
@@ -418,10 +418,10 @@ def _resolve_requirement(
 
             for not_found in self._resolve_requirement(
                 dep_requirement,
+                exclude_configuration,
                 resolved_dists_by_key,
                 required,
                 required_by=resolved_distribution.distribution,
-                exclude_configuration=exclude_configuration,
             ):
                 yield not_found
 
@@ -556,9 +556,9 @@ def record_unresolved(dist_not_found):
             with TRACER.timed("Resolving {}".format(qualified_req_or_not_found.requirement), V=2):
                 for not_found in self._resolve_requirement(
                     requirement=qualified_req_or_not_found.requirement,
+                    exclude_configuration=exclude_configuration,
                     required=qualified_req_or_not_found.required,
                     resolved_dists_by_key=resolved_dists_by_key,
-                    exclude_configuration=exclude_configuration,
                 ):
                     record_unresolved(not_found)
 

pex/pex_builder.py

@@ -29,7 +29,6 @@
 from pex.compiler import Compiler
 from pex.dist_metadata import Distribution, MetadataError
 from pex.enum import Enum
-from pex.environment import PEXEnvironment
 from pex.finders import get_entry_point_from_console_script, get_script_from_distributions
 from pex.interpreter import PythonInterpreter
 from pex.layout import Layout
@@ -374,21 +373,6 @@ def add_requirement(self, req):
         self._ensure_unfrozen("Adding a requirement")
         self._pex_info.add_requirement(req)
 
-    def add_from_requirements_pex(self, pex):
-        """Add requirements from an existing pex.
-
-        :param pex: The path to an existing .pex file or unzipped pex directory.
-        """
-        self._ensure_unfrozen("Adding from pex")
-        pex_info = PexInfo.from_pex(pex)
-        pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info)
-        for fingerprinted_dist in pex_environment.iter_distributions():
-            self.add_distribution(
-                dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint
-            )
-        for requirement in pex_info.requirements:
-            self.add_requirement(requirement)
-
     def set_executable(self, filename, env_filename=None):
         """Set the executable for this environment.
 

tests/integration/test_excludes.py

@@ -10,21 +10,22 @@
 from pex.executor import Executor
 from pex.pep_503 import ProjectName
 from pex.pex import PEX
+from pex.pex_info import PexInfo
 from pex.typing import TYPE_CHECKING
 from pex.venv.virtualenv import Virtualenv
 from testing import PY_VER, data, make_env, run_pex_command
 
 if TYPE_CHECKING:
-    from typing import Any
+    from typing import Any, Tuple
 
 
-@pytest.mark.skipif(PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13")
-def test_exclude(tmpdir):
-    # type: (Any) -> None
+@pytest.fixture(scope="module")
+def requests_certifi_excluded_pex(tmpdir_factory):
+    # type: (Any) -> str
 
     requests_lock = data.path("locks", "requests.lock.json")
-    pex_root = os.path.join(str(tmpdir), "pex_root")
-    pex = os.path.join(str(tmpdir), "pex")
+    pex_root = str(tmpdir_factory.mktemp("pex_root"))
+    pex = str(tmpdir_factory.mktemp("pex"))
     run_pex_command(
         args=[
             "--lock",
@@ -44,34 +45,100 @@ def test_exclude(tmpdir):
         dist.metadata.project_name for dist in PEX(pex).resolve()
     )
 
-    # The exclude option is buyer beware. A PEX using this option will not work if the excluded
-    # distributions carry modules that are, in fact, needed at run time.
-    requests_cmd = [pex, "-c", "import requests, sys; print(sys.modules['certifi'].__file__)"]
-    expected_import_error_msg = "ModuleNotFoundError: No module named 'certifi'"
+    return pex
+
+
+REQUESTS_CMD = ["-c", "import requests, sys; print(sys.modules['certifi'].__file__)"]
+EXPECTED_IMPORT_ERROR_MSG = "ModuleNotFoundError: No module named 'certifi'"
 
-    process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    _, stderr = process.communicate()
-    assert process.returncode != 0
 
-    assert expected_import_error_msg in stderr.decode("utf-8"), stderr.decode("utf-8")
+@pytest.fixture(scope="module")
+def certifi_venv(tmpdir_factory):
+    # type: (Any) -> Virtualenv
 
-    venv_dir = os.path.join(str(tmpdir), "venv")
-    venv = Virtualenv.create(venv_dir)
+    venv = Virtualenv.create(venv_dir=str(tmpdir_factory.mktemp("venv")))
     pip = venv.install_pip()
 
     # N.B.: The constraining lock requirement is the one expressed by requests: certifi>=2017.4.17
     # The actual locked version is 2023.7.22; so we stress this crease and use a different, but
     # allowed, version.
     subprocess.check_call(args=[pip, "install", "certifi==2017.4.17"])
 
+    return venv
+
+
+skip_unless_37_to_312 = pytest.mark.skipif(
+    PY_VER < (3, 7) or PY_VER >= (3, 13), reason="The lock used is for >=3.7,<3.13"
+)
+
+
+def assert_certifi_import_behavior(
+    pex,  # type: str
+    certifi_venv,  # type: Virtualenv
+):
+    requests_cmd = [pex] + REQUESTS_CMD
+
     # Although the venv has certifi available, a PEX is hermetic by default; so it shouldn't be
     # used.
     with pytest.raises(Executor.NonZeroExit) as exc:
-        venv.interpreter.execute(args=requests_cmd)
-    assert expected_import_error_msg in exc.value.stderr
+        certifi_venv.interpreter.execute(args=requests_cmd)
+    assert EXPECTED_IMPORT_ERROR_MSG in exc.value.stderr
 
     # Allowing the `sys.path` to be inherited should allow the certifi hole to be filled in.
-    _, stdout, _ = venv.interpreter.execute(
+    _, stdout, _ = certifi_venv.interpreter.execute(
         args=requests_cmd, env=make_env(PEX_INHERIT_PATH="fallback")
     )
-    assert venv.site_packages_dir == commonprefix([venv.site_packages_dir, stdout.strip()])
+    assert certifi_venv.site_packages_dir == commonprefix(
+        [certifi_venv.site_packages_dir, stdout.strip()]
+    )
+
+
+@skip_unless_37_to_312
+def test_exclude(
+    tmpdir,  # type: Any
+    requests_certifi_excluded_pex,  # type: str
+    certifi_venv,  # type: Virtualenv
+):
+    # type: (...) -> None
+
+    requests_cmd = [requests_certifi_excluded_pex] + REQUESTS_CMD
+
+    # The exclude option is buyer beware. A PEX using this option will not work if the excluded
+    # distributions carry modules that are, in fact, needed at run time.
+    process = subprocess.Popen(args=requests_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    _, stderr = process.communicate()
+    assert process.returncode != 0
+    assert EXPECTED_IMPORT_ERROR_MSG in stderr.decode("utf-8"), stderr.decode("utf-8")
+
+    assert_certifi_import_behavior(requests_certifi_excluded_pex, certifi_venv)
+
+
+@skip_unless_37_to_312
+def test_requirements_pex_exclude(
+    tmpdir,  # type: Any
+    requests_certifi_excluded_pex,  # type: str
+    certifi_venv,  # type: Virtualenv
+):
+    # type: (...) -> None
+
+    pex_root = PexInfo.from_pex(requests_certifi_excluded_pex).pex_root
+    pex = os.path.join(str(tmpdir), "pex")
+    run_pex_command(
+        args=[
+            "--requirements-pex",
+            requests_certifi_excluded_pex,
+            "ansicolors==1.1.8",
+            "-o",
+            pex,
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+        ]
+    ).assert_success()
+
+    # Shouldn't need the certifi hole filled to import colors.
+    output = subprocess.check_output(args=[pex, "-c", "import colors; print(colors.__file__)"])
+    assert pex_root == commonprefix([pex_root, output.decode("utf-8").strip()])
+
+    assert_certifi_import_behavior(pex, certifi_venv)

tests/test_pex_builder.py

@@ -210,43 +210,6 @@ def test_pex_builder_deterministic_timestamp():
             assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist())
 
 
-def test_pex_builder_from_requirements_pex():
-    # type: () -> None
-    def build_from_req_pex(path, req_pex):
-        # type: (str, str) -> PEXBuilder
-        pb = PEXBuilder(path=path)
-        pb.add_from_requirements_pex(req_pex)
-        with open(os.path.join(path, "exe.py"), "w") as fp:
-            fp.write(exe_main)
-        pb.set_executable(os.path.join(path, "exe.py"))
-        pb.freeze()
-        return pb
-
-    def verify(pb):
-        # type: (PEXBuilder) -> None
-        success_txt = os.path.join(pb.path(), "success.txt")
-        PEX(pb.path(), interpreter=pb.interpreter).run(args=[success_txt])
-        assert os.path.exists(success_txt)
-        with open(success_txt) as fp:
-            assert fp.read() == "success"
-
-    # Build from pex dir.
-    with temporary_dir() as td2:
-        with temporary_dir() as td1, make_bdist("p1") as p1:
-            pb1 = write_pex(td1, dists=[p1])
-            pb2 = build_from_req_pex(td2, pb1.path())
-        verify(pb2)
-
-    # Build from .pex file.
-    with temporary_dir() as td4:
-        with temporary_dir() as td3, make_bdist("p1") as p1:
-            pb3 = write_pex(td3, dists=[p1])
-            target = os.path.join(td3, "foo.pex")
-            pb3.build(target)
-            pb4 = build_from_req_pex(td4, target)
-        verify(pb4)
-
-
 def test_pex_builder_script_from_pex_path(tmpdir):
     # type: (Any) -> None