Implement support for --exclude <req>.

pex/bin/pex.py

@@ -24,6 +24,7 @@
     register_global_arguments,
 )
 from pex.common import die, is_pyc_dir, is_pyc_file, safe_mkdtemp
+from pex.dependency_manager import DependencyManager
 from pex.enum import Enum
 from pex.inherit_path import InheritPath
 from pex.interpreter_constraints import InterpreterConstraints
@@ -290,6 +291,22 @@ def configure_clp_pex_options(parser):
         ),
     )
 
+    group.add_argument(
+        "--exclude",
+        dest="excluded",
+        default=[],
+        type=str,
+        action="append",
+        help=(
+            "Specifies a requirement to exclude from the built PEX. Any distribution included in "
+            "the PEX's resolve that matches the requirement is excluded from the built PEX along "
+            "with all of its transitive dependencies that are not also required by other "
+            "non-excluded distributions. At runtime, the PEX will boot without checking the "
+            "excluded dependencies are available (say, via `--inherit-path`). This option can be "
+            "used multiple times."
+        ),
+    )
+
     group.add_argument(
         "--compile",
         "--no-compile",
@@ -808,11 +825,15 @@ def build_pex(
     pex_info.strip_pex_env = options.strip_pex_env
     pex_info.interpreter_constraints = interpreter_constraints
 
-    for requirements_pex in options.requirements_pexes:
-        pex_builder.add_from_requirements_pex(requirements_pex)
+    dependency_manager = DependencyManager()
+    with TRACER.timed(
+        "Adding distributions from pexes: {}".format(" ".join(options.requirements_pexes))
+    ):
+        for requirements_pex in options.requirements_pexes:
+            dependency_manager.add_from_pex(requirements_pex)
 
     with TRACER.timed(
-        "Resolving distributions ({})".format(
+        "Resolving distributions for requirements: {}".format(
             " ".join(
                 itertools.chain.from_iterable(
                     (
@@ -824,22 +845,21 @@ def build_pex(
         )
     ):
         try:
-            result = resolve(
-                targets=targets,
-                requirement_configuration=requirement_configuration,
-                resolver_configuration=resolver_configuration,
-                compile_pyc=options.compile,
-                ignore_errors=options.ignore_errors,
-            )
-            for installed_dist in result.installed_distributions:
-                pex_builder.add_distribution(
-                    installed_dist.distribution, fingerprint=installed_dist.fingerprint
+            dependency_manager.add_from_installed(
+                resolve(
+                    targets=targets,
+                    requirement_configuration=requirement_configuration,
+                    resolver_configuration=resolver_configuration,
+                    compile_pyc=options.compile,
+                    ignore_errors=options.ignore_errors,
                 )
-                for direct_req in installed_dist.direct_requirements:
-                    pex_builder.add_requirement(direct_req)
+            )
         except Unsatisfiable as e:
             die(str(e))
 
+    with TRACER.timed("Configuring PEX dependencies"):
+        dependency_manager.configure(pex_builder, excluded=options.excluded)
+
     if options.entry_point:
         pex_builder.set_entry_point(options.entry_point)
     elif options.script:

pex/dependency_manager.py

@@ -0,0 +1,108 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+from collections import defaultdict
+
+from pex import pex_warnings
+from pex.dist_metadata import Requirement
+from pex.environment import PEXEnvironment
+from pex.exclude_configuration import ExcludeConfiguration
+from pex.fingerprinted_distribution import FingerprintedDistribution
+from pex.orderedset import OrderedSet
+from pex.pep_503 import ProjectName
+from pex.pex_builder import PEXBuilder
+from pex.pex_info import PexInfo
+from pex.resolve.resolvers import Installed
+from pex.tracer import TRACER
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import DefaultDict, Iterable, Iterator
+
+    import attr  # vendor:skip
+else:
+    from pex.third_party import attr
+
+
+@attr.s
+class DependencyManager(object):
+    _requirements = attr.ib(factory=OrderedSet)  # type: OrderedSet[Requirement]
+    _distributions = attr.ib(factory=OrderedSet)  # type: OrderedSet[FingerprintedDistribution]
+
+    def add_from_pex(self, pex):
+        # type: (str) -> None
+
+        pex_info = PexInfo.from_pex(pex)
+        self._requirements.update(Requirement.parse(req) for req in pex_info.requirements)
+
+        pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info)
+        self._distributions.update(pex_environment.iter_distributions())
+
+    def add_from_installed(self, installed):
+        # type: (Installed) -> None
+
+        for installed_dist in installed.installed_distributions:
+            self._requirements.update(installed_dist.direct_requirements)
+            self._distributions.add(installed_dist.fingerprinted_distribution)
+
+    def configure(
+        self,
+        pex_builder,  # type: PEXBuilder
+        excluded=(),  # type: Iterable[str]
+    ):
+        # type: (...) -> None
+
+        exclude_configuration = ExcludeConfiguration.create(excluded)
+        exclude_configuration.configure(pex_builder.info)
+
+        dists_by_project_name = defaultdict(
+            OrderedSet
+        )  # type: DefaultDict[ProjectName, OrderedSet[FingerprintedDistribution]]
+        for dist in self._distributions:
+            dists_by_project_name[dist.distribution.metadata.project_name].add(dist)
+
+        def iter_non_excluded_distributions(requirements):
+            # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution]
+            for req in requirements:
+                candidate_dists = dists_by_project_name[req.project_name]
+                for candidate_dist in tuple(candidate_dists):
+                    if candidate_dist.distribution not in req:
+                        continue
+                    candidate_dists.discard(candidate_dist)
+
+                    excluded_by = exclude_configuration.excluded_by(candidate_dist.distribution)
+                    if excluded_by:
+                        excludes = " and ".join(map(str, excluded_by))
+                        TRACER.log(
+                            "Skipping adding {candidate}: excluded by {excludes}".format(
+                                candidate=candidate_dist.distribution, excludes=excludes
+                            )
+                        )
+                        for root_req in self._requirements:
+                            if candidate_dist.distribution in root_req:
+                                pex_warnings.warn(
+                                    "The distribution {dist} was required by the input requirement "
+                                    "{root_req} but excluded by configured excludes: "
+                                    "{excludes}".format(
+                                        dist=candidate_dist.distribution,
+                                        root_req=root_req,
+                                        excludes=excludes,
+                                    )
+                                )
+                        continue
+
+                    yield candidate_dist
+                    for dep in iter_non_excluded_distributions(
+                        candidate_dist.distribution.requires()
+                    ):
+                        yield dep
+
+        for fingerprinted_dist in iter_non_excluded_distributions(self._requirements):
+            pex_builder.add_distribution(
+                dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint
+            )
+
+        for requirement in self._requirements:
+            pex_builder.add_requirement(requirement)

pex/environment.py

@@ -12,6 +12,7 @@
 from pex import dist_metadata, pex_warnings, targets
 from pex.common import pluralize
 from pex.dist_metadata import Distribution, Requirement
+from pex.exclude_configuration import ExcludeConfiguration
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.inherit_path import InheritPath
 from pex.interpreter import PythonInterpreter
@@ -348,12 +349,23 @@ def _resolve_requirement(
         resolved_dists_by_key,  # type: MutableMapping[_RequirementKey, FingerprintedDistribution]
         required,  # type: bool
         required_by=None,  # type: Optional[Distribution]
+        exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
     ):
         # type: (...) -> Iterator[_DistributionNotFound]
         requirement_key = _RequirementKey.create(requirement)
         if requirement_key in resolved_dists_by_key:
             return
 
+        excluded_by = exclude_configuration.excluded_by(requirement)
+        if excluded_by:
+            TRACER.log(
+                "Skipping resolving {requirement}: excluded by {excludes}".format(
+                    requirement=requirement,
+                    excludes=" and ".join(map(str, excluded_by)),
+                )
+            )
+            return
+
         available_distributions = [
             ranked_dist
             for ranked_dist in self._available_ranked_dists_by_project_name[
@@ -409,6 +421,7 @@ def _resolve_requirement(
                 resolved_dists_by_key,
                 required,
                 required_by=resolved_distribution.distribution,
+                exclude_configuration=exclude_configuration,
             ):
                 yield not_found
 
@@ -502,14 +515,21 @@ def resolve(self):
         # type: () -> Iterable[Distribution]
         if self._resolved_dists is None:
             all_reqs = [Requirement.parse(req) for req in self._pex_info.requirements]
+            exclude_configuration = ExcludeConfiguration.create(excluded=self._pex_info.excluded)
             self._resolved_dists = tuple(
                 fingerprinted_distribution.distribution
-                for fingerprinted_distribution in self.resolve_dists(all_reqs)
+                for fingerprinted_distribution in self.resolve_dists(
+                    all_reqs, exclude_configuration=exclude_configuration
+                )
             )
         return self._resolved_dists
 
-    def resolve_dists(self, reqs):
-        # type: (Iterable[Requirement]) -> Iterable[FingerprintedDistribution]
+    def resolve_dists(
+        self,
+        reqs,  # type: Iterable[Requirement]
+        exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
+    ):
+        # type: (...) -> Iterable[FingerprintedDistribution]
 
         self._update_candidate_distributions(self.iter_distributions())
 
@@ -538,6 +558,7 @@ def record_unresolved(dist_not_found):
                     requirement=qualified_req_or_not_found.requirement,
                     required=qualified_req_or_not_found.required,
                     resolved_dists_by_key=resolved_dists_by_key,
+                    exclude_configuration=exclude_configuration,
                 ):
                     record_unresolved(not_found)
 

pex/exclude_configuration.py

@@ -0,0 +1,36 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+from pex.dist_metadata import Distribution, Requirement
+from pex.pex_info import PexInfo
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Iterable, Tuple, Union
+
+    import attr  # vendor:skip
+else:
+    from pex.third_party import attr
+
+
+@attr.s(frozen=True)
+class ExcludeConfiguration(object):
+    @classmethod
+    def create(cls, excluded):
+        # type: (Iterable[str]) -> ExcludeConfiguration
+        return cls(excluded=tuple(Requirement.parse(req) for req in excluded))
+
+    _excluded = attr.ib(factory=tuple)  # type: Tuple[Requirement, ...]
+
+    def configure(self, pex_info):
+        # type: (PexInfo) -> None
+        for excluded in self._excluded:
+            pex_info.add_excluded(excluded)
+
+    def excluded_by(self, item):
+        # type: (Union[Distribution, Requirement]) -> Iterable[Requirement]
+        if isinstance(item, Distribution):
+            return tuple(req for req in self._excluded if item in req)
+        return tuple(req for req in self._excluded if item.project_name == req.project_name)

pex/pex_info.py

@@ -21,6 +21,8 @@
 if TYPE_CHECKING:
     from typing import Any, Dict, Iterable, Mapping, Optional, Text, Tuple, Union
 
+    from pex.dist_metadata import Requirement
+
     # N.B.: These are expensive imports and PexInfo is used during PEX bootstrapping which we want
     # to be as fast as possible.
     from pex.interpreter import PythonInterpreter
@@ -144,6 +146,8 @@ def __init__(self, info=None):
             raise ValueError("Expected requirements to be a list, got %s" % type(requirements))
         self._requirements = OrderedSet(self._parse_requirement_tuple(req) for req in requirements)
 
+        self._excluded = OrderedSet(self._pex_info.get("excluded", ()))  # type: OrderedSet[str]
+
     def _get_safe(self, key):
         if key not in self._pex_info:
             return None
@@ -445,6 +449,15 @@ def add_requirement(self, requirement):
     def requirements(self):
         return self._requirements
 
+    def add_excluded(self, requirement):
+        # type: (Requirement) -> None
+        self._excluded.add(str(requirement))
+
+    @property
+    def excluded(self):
+        # type: () -> Iterable[str]
+        return self._excluded
+
     def add_distribution(self, location, sha):
         self._distributions[location] = sha
 
@@ -527,12 +540,14 @@ def update(self, other):
                 other.interpreter_constraints
             )
         self._requirements.update(other.requirements)
+        self._excluded.update(other.excluded)
 
     def as_json_dict(self):
         # type: () -> Dict[str, Any]
         data = self._pex_info.copy()
         data["inherit_path"] = self.inherit_path.value
         data["requirements"] = list(self._requirements)
+        data["excluded"] = list(self._excluded)
         data["interpreter_constraints"] = [str(ic) for ic in self.interpreter_constraints]
         data["distributions"] = self._distributions.copy()
         return data
@@ -541,6 +556,7 @@ def dump(self):
         # type: (...) -> str
         data = self.as_json_dict()
         data["requirements"].sort()
+        data["excluded"].sort()
         data["interpreter_constraints"].sort()
         return json.dumps(data, sort_keys=True)
 

testing/data/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import os.path
+import pkgutil
+
+
+def load(rel_path):
+    # type: (str) -> bytes
+    data = pkgutil.get_data(__name__, rel_path)
+    if data is None:
+        raise ValueError(
+            "No resource found at {rel_path} from package {name}.".format(
+                rel_path=rel_path, name=__name__
+            )
+        )
+    return data
+
+
+def path(*rel_path):
+    # type: (*str) -> str
+    path = os.path.join(os.path.dirname(__file__), *rel_path)
+    if not os.path.isfile(path):
+        raise ValueError("No resource found at {path}.".format(path=path))
+    return path

tests/data/platforms/__init__.py → testing/data/locks/__init__.py

@@ -1,2 +1,2 @@
-# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).