Add blacklist handling for skipping requirements in pex resolver

pex/resolver.py

@@ -151,10 +151,11 @@ def filter_packages_by_interpreter(cls, packages, interpreter, platform):
     return [package for package in packages
         if package.compatible(interpreter.identity, platform)]
 
-  def __init__(self, allow_prereleases=None, interpreter=None, platform=None):
+  def __init__(self, allow_prereleases=None, interpreter=None, platform=None, pkg_blacklist=None):
     self._interpreter = interpreter or PythonInterpreter.get()
     self._platform = platform or Platform.current()
     self._allow_prereleases = allow_prereleases
+    self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {}
 
   def package_iterator(self, resolvable, existing=None):
     if existing:
@@ -180,6 +181,12 @@ def build(self, package, options):
         'Could not get distribution for %s on platform %s.' % (package, self._platform))
     return dist
 
+  def _resolvable_is_blacklisted(self, resolvable_name):
+    return (
+      resolvable_name in self._blacklist and
+      self._interpreter.identity.matches(self._blacklist[resolvable_name])
+    )
+
   def resolve(self, resolvables, resolvable_set=None):
     resolvables = [(resolvable, None) for resolvable in resolvables]
     resolvable_set = resolvable_set or _ResolvableSet()
@@ -193,7 +200,11 @@ def resolve(self, resolvables, resolvable_set=None):
         if resolvable in processed_resolvables:
           continue
         packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name))
-        resolvable_set.merge(resolvable, packages, parent)
+
+        # TODO: Remove blacklist strategy in favor of smart requirement handling
+        # https://github.com/pantsbuild/pex/issues/456
+        if not self._resolvable_is_blacklisted(resolvable.name):
+          resolvable_set.merge(resolvable, packages, parent)
         processed_resolvables.add(resolvable)
 
       built_packages = {}
@@ -299,7 +310,8 @@ def resolve(requirements,
             precedence=None,
             cache=None,
             cache_ttl=None,
-            allow_prereleases=None):
+            allow_prereleases=None,
+            pkg_blacklist=None):
   """Produce all distributions needed to (recursively) meet `requirements`
 
   :param requirements: An iterator of Requirement-like things, either
@@ -329,6 +341,14 @@ def resolve(requirements,
     ``context``.
   :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
     unspecified only stable versions will be resolved, unless explicitly included.
+  :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
+    an interpreter constraint. If a package name is in the blacklist and its interpreter
+    constraint matches the target interpreter, skip the requirement. This is needed to ensure
+    that universal requirement resolves for a target interpreter version do not error out on
+    interpreter specific requirements such as backport libs like `functools32`.
+    For example, a valid blacklist is {'functools32': 'CPython>3'}.
+    NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
+    tracked by: https://github.com/pantsbuild/pex/issues/456
   :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``.
   :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
   :raises Untranslateable: If no compatible distributions could be acquired for
@@ -367,11 +387,13 @@ def resolve(requirements,
                                cache_ttl,
                                allow_prereleases=allow_prereleases,
                                interpreter=interpreter,
-                               platform=platform)
+                               platform=platform,
+                               pkg_blacklist=pkg_blacklist)
   else:
     resolver = Resolver(allow_prereleases=allow_prereleases,
                         interpreter=interpreter,
-                        platform=platform)
+                        platform=platform,
+                        pkg_blacklist=pkg_blacklist)
 
   return resolver.resolve(resolvables_from_iterable(requirements, builder))
 
@@ -384,7 +406,8 @@ def resolve_multi(requirements,
                   precedence=None,
                   cache=None,
                   cache_ttl=None,
-                  allow_prereleases=None):
+                  allow_prereleases=None,
+                  pkg_blacklist=None):
   """A generator function that produces all distributions needed to meet `requirements`
   for multiple interpreters and/or platforms.
 
@@ -415,6 +438,14 @@ def resolve_multi(requirements,
     ``context``.
   :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
     unspecified only stable versions will be resolved, unless explicitly included.
+  :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
+    an interpreter constraint. If a package name is in the blacklist and its interpreter
+    constraint matches the target interpreter, skip the requirement. This is needed to ensure
+    that universal requirement resolves for a target interpreter version do not error out on
+    interpreter specific requirements such as backport libs like `functools32`.
+    For example, a valid blacklist is {'functools32': 'CPython>3'}.
+    NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
+    tracked by: https://github.com/pantsbuild/pex/issues/456
   :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``.
   :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
   :raises Untranslateable: If no compatible distributions could be acquired for
@@ -435,7 +466,8 @@ def resolve_multi(requirements,
                                 precedence,
                                 cache,
                                 cache_ttl,
-                                allow_prereleases):
+                                allow_prereleases,
+                                pkg_blacklist=pkg_blacklist):
         if resolvable not in seen:
           seen.add(resolvable)
           yield resolvable

tests/test_resolver.py

@@ -2,17 +2,20 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
+import sys
 import time
 
 import pytest
 from twitter.common.contextutil import temporary_dir
 
 from pex.common import safe_copy
+from pex.compatibility import PY2
 from pex.crawler import Crawler
 from pex.fetcher import Fetcher
+from pex.interpreter import PythonInterpreter
 from pex.package import EggPackage, SourcePackage
 from pex.resolvable import ResolvableRequirement
-from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi
+from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve, resolve_multi
 from pex.resolver_options import ResolverOptionsBuilder
 from pex.testing import make_sdist
 
@@ -349,3 +352,26 @@ def test_resolvable_set_built():
   updated_rs.merge(rq, [binary_pkg])
   assert updated_rs.get('foo') == set([binary_pkg])
   assert updated_rs.packages() == [(rq, set([binary_pkg]), None, False)]
+
+
+def test_resolver_blacklist():
+  if PY2:
+    blacklist = {'project2': '<3'}
+    required_project = "project2;python_version>'3'"
+  else:
+    blacklist = {'project2': '>3'}
+    required_project = "project2;python_version<'3'"
+
+  project1 = make_sdist(name='project1', version='1.0.0', install_reqs=[required_project])
+  project2 = make_sdist(name='project2', version='1.1.0')
+
+  with temporary_dir() as td:
+    safe_copy(project1, os.path.join(td, os.path.basename(project1)))
+    safe_copy(project2, os.path.join(td, os.path.basename(project2)))
+    fetchers = [Fetcher([td])]
+
+    dists = resolve(['project1'], fetchers=fetchers)
+    assert len(dists) == 2
+
+    dists = resolve(['project1'], fetchers=fetchers, pkg_blacklist=blacklist)
+    assert len(dists) == 1