Fix --exclude. (#2409)

Previously, `--exclude`s were processed after all resolution was 
complete. Not only was this sub-optimal, incurring more resolution work
(downloads, backtracks, etc.), but it also could lead to failed
interactions (metadata extraction & wheel builds) with excluded
distributions.

Fix this by plumbing `--exclude`s all the way through the Pip resolve
process. Additionally, fix the existing plumbing through the two other
resolve processes: PEX repository resolution and lock file resolution.

Fixes #455
Fixes #2097

.gitignore

@@ -1,5 +1,7 @@
 *.pyc
 *.pyo
+__pycache__/
+
 *~
 
 /.mypy_cache/

pex/bin/pex.py

@@ -27,6 +27,7 @@
 from pex.dependency_manager import DependencyManager
 from pex.docs.command import serve_html_docs
 from pex.enum import Enum
+from pex.exclude_configuration import ExcludeConfiguration
 from pex.inherit_path import InheritPath
 from pex.interpreter_constraints import InterpreterConstraints
 from pex.layout import Layout, ensure_installed
@@ -897,6 +898,7 @@ def build_pex(
             )
             excluded.extend(requirements_pex_info.excluded)
 
+    exclude_configuration = ExcludeConfiguration.create(excluded)
     with TRACER.timed(
         "Resolving distributions for requirements: {}".format(
             " ".join(
@@ -922,13 +924,14 @@ def build_pex(
                         if options.pre_install_wheels
                         else InstallableType.WHEEL_FILE
                     ),
-                )
+                    exclude_configuration=exclude_configuration,
+                ),
             )
         except Unsatisfiable as e:
             die(str(e))
 
     with TRACER.timed("Configuring PEX dependencies"):
-        dependency_manager.configure(pex_builder, excluded=excluded)
+        dependency_manager.configure(pex_builder, exclude_configuration=exclude_configuration)
 
     if options.entry_point:
         pex_builder.set_entry_point(options.entry_point)

pex/build_system/pep_517.py

@@ -53,7 +53,7 @@ def _default_build_system(
                     )
                 )
                 resolved_reqs.add("setuptools")
-                extra_env.update(__PEX_UNVENDORED__="1")
+                extra_env.update(__PEX_UNVENDORED__="setuptools")
             else:
                 requires = [
                     selected_pip_version.setuptools_requirement,

pex/cli/commands/lock.py

@@ -12,7 +12,6 @@
 
 from pex import pex_warnings
 from pex.argparse import HandleBoolAction
-from pex.asserts import production_assert
 from pex.cli.command import BuildTimeCommand
 from pex.commands.command import JsonMixin, OutputMixin
 from pex.common import is_exe, pluralize, safe_delete, safe_open
@@ -25,6 +24,7 @@
     RequirementParseError,
 )
 from pex.enum import Enum
+from pex.exceptions import production_assert
 from pex.interpreter import PythonInterpreter
 from pex.pep_376 import InstalledWheel, Record
 from pex.pep_427 import InstallableType
@@ -1162,7 +1162,7 @@ def _process_lock_update(
         updates = []  # type: List[Union[DeleteUpdate, VersionUpdate, ArtifactsUpdate]]
         warnings = []  # type: List[str]
         for resolve_update in lock_update.resolves:
-            platform = resolve_update.updated_resolve.platform_tag or "universal"
+            platform = resolve_update.updated_resolve.target_platform
             if not resolve_update.updates:
                 print(
                     "No updates for lock generated by {platform}.".format(platform=platform),
@@ -1527,7 +1527,7 @@ def _sync(self):
                         "Would lock {count} {project} for platform {platform}:".format(
                             count=len(locked_resolve.locked_requirements),
                             project=pluralize(locked_resolve.locked_requirements, "project"),
-                            platform=locked_resolve.platform_tag or "universal",
+                            platform=locked_resolve.target_platform,
                         ),
                         file=output,
                     )

pex/dependency_manager.py

@@ -6,20 +6,21 @@
 from collections import defaultdict
 
 from pex import pex_warnings
+from pex.common import pluralize
 from pex.dist_metadata import Requirement
 from pex.environment import PEXEnvironment
+from pex.exceptions import production_assert
 from pex.exclude_configuration import ExcludeConfiguration
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.orderedset import OrderedSet
 from pex.pep_503 import ProjectName
 from pex.pex_builder import PEXBuilder
 from pex.pex_info import PexInfo
 from pex.resolve.resolvers import ResolveResult
-from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import DefaultDict, Iterable, Iterator
+    from typing import DefaultDict
 
     import attr  # vendor:skip
 else:
@@ -58,64 +59,42 @@ def add_from_resolved(self, resolved):
     def configure(
         self,
         pex_builder,  # type: PEXBuilder
-        excluded=(),  # type: Iterable[str]
+        exclude_configuration=ExcludeConfiguration(),  # type: ExcludeConfiguration
     ):
         # type: (...) -> None
 
-        exclude_configuration = ExcludeConfiguration.create(excluded)
         exclude_configuration.configure(pex_builder.info)
 
-        dists_by_project_name = defaultdict(
-            OrderedSet
-        )  # type: DefaultDict[ProjectName, OrderedSet[FingerprintedDistribution]]
-        for dist in self._distributions:
-            dists_by_project_name[dist.distribution.metadata.project_name].add(dist)
-
         root_requirements_by_project_name = defaultdict(
             OrderedSet
         )  # type: DefaultDict[ProjectName, OrderedSet[Requirement]]
         for root_req in self._requirements:
             root_requirements_by_project_name[root_req.project_name].add(root_req)
 
-        def iter_non_excluded_distributions(requirements):
-            # type: (Iterable[Requirement]) -> Iterator[FingerprintedDistribution]
-            for req in requirements:
-                candidate_dists = dists_by_project_name[req.project_name]
-                for candidate_dist in tuple(candidate_dists):
-                    if candidate_dist.distribution not in req:
-                        continue
-                    candidate_dists.discard(candidate_dist)
-
-                    excluded_by = exclude_configuration.excluded_by(candidate_dist.distribution)
-                    if excluded_by:
-                        excludes = " and ".join(map(str, excluded_by))
-                        TRACER.log(
-                            "Skipping adding {candidate}: excluded by {excludes}".format(
-                                candidate=candidate_dist.distribution, excludes=excludes
-                            )
-                        )
-                        for root_req in root_requirements_by_project_name[
-                            candidate_dist.distribution.metadata.project_name
-                        ]:
-                            if candidate_dist.distribution in root_req:
-                                pex_warnings.warn(
-                                    "The distribution {dist} was required by the input requirement "
-                                    "{root_req} but excluded by configured excludes: "
-                                    "{excludes}".format(
-                                        dist=candidate_dist.distribution,
-                                        root_req=root_req,
-                                        excludes=excludes,
-                                    )
-                                )
-                        continue
-
-                    yield candidate_dist
-                    for dep in iter_non_excluded_distributions(
-                        candidate_dist.distribution.requires()
-                    ):
-                        yield dep
-
-        for fingerprinted_dist in iter_non_excluded_distributions(self._requirements):
+        for fingerprinted_dist in self._distributions:
+            excluded_by = exclude_configuration.excluded_by(fingerprinted_dist.distribution)
+            if excluded_by:
+                excludes = " and ".join(map(str, excluded_by))
+                root_reqs = root_requirements_by_project_name[fingerprinted_dist.project_name]
+                production_assert(
+                    len(root_reqs) > 0,
+                    "The deep --exclude mechanism failed to exclude {dist} from transitive "
+                    "requirements. It should have been excluded by configured excludes: "
+                    "{excludes} but was not.".format(
+                        dist=fingerprinted_dist.distribution, excludes=excludes
+                    ),
+                )
+                pex_warnings.warn(
+                    "The distribution {dist} was required by the input {requirements} "
+                    "{root_reqs} but ultimately excluded by configured excludes: "
+                    "{excludes}".format(
+                        dist=fingerprinted_dist.distribution,
+                        requirements=pluralize(root_reqs, "requirement"),
+                        root_reqs=" and ".join(map(str, root_reqs)),
+                        excludes=excludes,
+                    )
+                )
+                continue
             pex_builder.add_distribution(
                 dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint
             )

pex/dist_metadata.py

@@ -20,7 +20,7 @@
 from io import StringIO
 from textwrap import dedent
 
-from pex import pex_warnings
+from pex import pex_warnings, specifier_sets
 from pex.common import open_zip, pluralize
 from pex.compatibility import to_unicode
 from pex.enum import Enum
@@ -681,19 +681,23 @@ def key(self):
         return self.project_name.normalized
 
     def __contains__(self, item):
-        # type: (Union[str, Version, Distribution, ProjectNameAndVersion]) -> bool
+        # type: (Union[str, Version, Distribution, ProjectNameAndVersion, Constraint]) -> bool
 
         # We emulate pkg_resources.Requirement.__contains__ pre-release behavior here since the
         # codebase expects it.
         return self.contains(item, prereleases=True)
 
     def contains(
         self,
-        item,  # type: Union[str, Version, Distribution, ProjectNameAndVersion]
+        item,  # type: Union[str, Version, Distribution, ProjectNameAndVersion, Constraint]
         prereleases=None,  # type: Optional[bool]
     ):
         # type: (...) -> bool
-        if isinstance(item, ProjectNameAndVersion):
+        if isinstance(item, Constraint):
+            return item.project_name == self.project_name and specifier_sets.includes(
+                self.specifier, item.specifier
+            )
+        elif isinstance(item, ProjectNameAndVersion):
             if item.canonicalized_project_name != self.project_name:
                 return False
             version = (

pex/environment.py

@@ -459,8 +459,12 @@ def _resolve_requirement(
             ):
                 yield not_found
 
-    def _root_requirements_iter(self, reqs):
-        # type: (Iterable[Requirement]) -> Iterator[QualifiedRequirementOrNotFound]
+    def _root_requirements_iter(
+        self,
+        reqs,  # type: Iterable[Requirement]
+        exclude_configuration,  # type: ExcludeConfiguration
+    ):
+        # type: (...) -> Iterator[QualifiedRequirementOrNotFound]
 
         # We want to pick one requirement for each key (required project) to then resolve
         # recursively.
@@ -477,6 +481,16 @@ def _root_requirements_iter(self, reqs):
             OrderedDict()
         )  # type: OrderedDict[ProjectName, List[_QualifiedRequirement]]
         for req in reqs:
+            excluded_by = exclude_configuration.excluded_by(req)
+            if excluded_by:
+                TRACER.log(
+                    "Skipping resolving {requirement}: excluded by {excludes}".format(
+                        requirement=req,
+                        excludes=" and ".join(map(str, excluded_by)),
+                    )
+                )
+                continue
+
             required = self._evaluate_marker(req)
             if not required:
                 continue
@@ -591,7 +605,7 @@ def record_unresolved(dist_not_found):
         resolved_dists_by_key = (
             OrderedDict()
         )  # type: OrderedDict[_RequirementKey, FingerprintedDistribution]
-        for qualified_req_or_not_found in self._root_requirements_iter(reqs):
+        for qualified_req_or_not_found in self._root_requirements_iter(reqs, exclude_configuration):
             if isinstance(qualified_req_or_not_found, _DistributionNotFound):
                 record_unresolved(qualified_req_or_not_found)
                 continue

pex/asserts.py → pex/exceptions.py

@@ -13,11 +13,11 @@
 _ASSERT_DETAILS = (
     dedent(
         """\
-    Pex {version}
-    platform: {platform}
-    python: {python_version}
-    argv: {argv}
-    """
+        Pex {version}
+        platform: {platform}
+        python: {python_version}
+        argv: {argv}
+        """
     )
     .format(
         version=__version__, platform=platform.platform(), python_version=sys.version, argv=sys.argv
@@ -27,23 +27,20 @@
 
 _ASSERT_ADVICE = dedent(
     """\
-    The error reported above resulted from an unexpected programming error which 
-    you should never encounter.
-    
+    The error reported above resulted from an unexpected error which you should
+    never encounter.
+
     Firstly, please accept our apology!
-    
+
     If you could file an issue with the error and details above, we'd be
     grateful. You can do that at https://github.com/pex-tool/pex/issues/new and
     redact or amend any details that expose sensitive information.
     """
 ).strip()
 
 
-def production_assert(condition, msg=""):
-    # type: (...) -> None
-
-    if condition:
-        return
+def reportable_unexpected_error_msg(msg=""):
+    # type: (str) -> str
 
     message = [msg, "---", _ASSERT_DETAILS]
     pex = os.environ.get("PEX")
@@ -64,4 +61,14 @@ def production_assert(condition, msg=""):
     message.append("---")
     message.append(_ASSERT_ADVICE)
 
-    raise AssertionError("\n".join(message))
+    return "\n".join(message)
+
+
+def production_assert(
+    condition,  # type: bool
+    msg="",  # type: str
+):
+    # type: (...) -> None
+
+    if not condition:
+        raise AssertionError(reportable_unexpected_error_msg(msg=msg))

pex/exclude_configuration.py

@@ -8,7 +8,7 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Iterable, Tuple, Union
+    from typing import Iterable, Iterator, Tuple, Union
 
     import attr  # vendor:skip
 else:
@@ -22,15 +22,24 @@ def create(cls, excluded):
         # type: (Iterable[str]) -> ExcludeConfiguration
         return cls(excluded=tuple(Requirement.parse(req) for req in excluded))
 
-    _excluded = attr.ib(factory=tuple)  # type: Tuple[Requirement, ...]
+    _excluded = attr.ib(default=())  # type: Tuple[Requirement, ...]
 
     def configure(self, pex_info):
         # type: (PexInfo) -> None
         for excluded in self._excluded:
             pex_info.add_excluded(excluded)
 
     def excluded_by(self, item):
-        # type: (Union[Distribution, Requirement]) -> Iterable[Requirement]
-        if isinstance(item, Distribution):
-            return tuple(req for req in self._excluded if item in req)
-        return tuple(req for req in self._excluded if item.project_name == req.project_name)
+        # type: (Union[Distribution, Requirement]) -> Tuple[Requirement, ...]
+        return tuple(req for req in self._excluded if item in req)
+
+    def __iter__(self):
+        # type: () -> Iterator[Requirement]
+        return iter(self._excluded)
+
+    def __bool__(self):
+        # type: () -> bool
+        return bool(self._excluded)
+
+    # N.B.: For Python 2.7.
+    __nonzero__ = __bool__