Add first-class support for multi-interpreter and multi-platform pex …

…construction. (#394)

This revives and extends the prior attempt in #201 to forklift the multi-interpreter/platform resolve behavior from pants directly into pex to better support general usage via pex(1) (and transitively, the Heron team's Bazel+pex integration).

Fixes #68.

.gitignore

@@ -1,6 +1,7 @@
 *~
 *.pyc
 *.egg-info
+/.cache
 /venv
 /build/*
 /dist/*

pex/bin/pex.py

@@ -30,7 +30,7 @@
 from pex.platforms import Platform
 from pex.requirements import requirements_from_file
 from pex.resolvable import Resolvable
-from pex.resolver import CachingResolver, Resolver, Unsatisfiable
+from pex.resolver import Unsatisfiable, resolve_multi
 from pex.resolver_options import ResolverOptionsBuilder
 from pex.tracer import TRACER
 from pex.variables import ENV, Variables
@@ -274,9 +274,12 @@ def configure_clp_pex_environment(parser):
   group.add_option(
       '--python',
       dest='python',
-      default=None,
+      default=[],
+      type='str',
+      action='append',
       help='The Python interpreter to use to build the pex.  Either specify an explicit '
-           'path to an interpreter, or specify a binary accessible on $PATH. '
+           'path to an interpreter, or specify a binary accessible on $PATH. This option '
+           'can be passed multiple times to create a multi-interpreter compatible pex. '
            'Default: Use current interpreter.')
 
   group.add_option(
@@ -290,15 +293,18 @@ def configure_clp_pex_environment(parser):
   group.add_option(
       '--platform',
       dest='platform',
-      default=Platform.current(),
-      help='The platform for which to build the PEX.  Default: %default')
+      default=[],
+      type=str,
+      action='append',
+      help='The platform for which to build the PEX. This option can be passed multiple times '
+           'to create a multi-platform compatible pex. Default: current platform.')
 
   group.add_option(
       '--interpreter-cache-dir',
       dest='interpreter_cache_dir',
       default='{pex_root}/interpreters',
       help='The interpreter cache to use for keeping track of interpreter dependencies '
-           'for the pex tool. [Default: ~/.pex/interpreters]')
+           'for the pex tool. Default: `~/.pex/interpreters`.')
 
   parser.add_option_group(group)
 
@@ -460,39 +466,54 @@ def installer_provider(sdist):
     return interpreter.with_extra(egg.name, egg.raw_version, egg.path)
 
 
-def interpreter_from_options(options):
+def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel):
   interpreter = None
 
-  if options.python:
-    if os.path.exists(options.python):
-      interpreter = PythonInterpreter.from_binary(options.python)
+  if python_interpreter:
+    if os.path.exists(python_interpreter):
+      interpreter = PythonInterpreter.from_binary(python_interpreter)
     else:
-      interpreter = PythonInterpreter.from_env(options.python)
+      interpreter = PythonInterpreter.from_env(python_interpreter)
     if interpreter is None:
-      die('Failed to find interpreter: %s' % options.python)
+      die('Failed to find interpreter: %s' % python_interpreter)
   else:
     interpreter = PythonInterpreter.get()
 
   with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
-    resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos)
+    resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos)
 
     # resolve setuptools
     interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
 
     # possibly resolve wheel
-    if interpreter and options.use_wheel:
+    if interpreter and use_wheel:
       interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
 
     return interpreter
 
 
-def build_pex(args, options, resolver_option_builder):
-  with TRACER.timed('Resolving interpreter', V=2):
-    interpreter = interpreter_from_options(options)
+def _lowest_version_interpreter(interpreters):
+  """Given a list of interpreters, return the one with the lowest version."""
+  lowest = interpreters[0]
+  for i in interpreters[1:]:
+    lowest = lowest if lowest < i else i
+  return lowest
 
-  if interpreter is None:
+
+def build_pex(args, options, resolver_option_builder):
+  with TRACER.timed('Resolving interpreters', V=2):
+    interpreters = [
+      get_interpreter(interpreter,
+                      options.interpreter_cache_dir,
+                      options.repos,
+                      options.use_wheel)
+      for interpreter in options.python or [None]
+    ]
+
+  if not interpreters:
     die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
 
+  interpreter = _lowest_version_interpreter(interpreters)
   pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter)
 
   pex_info = pex_builder.info
@@ -515,24 +536,21 @@ def build_pex(args, options, resolver_option_builder):
       constraints.append(r)
     resolvables.extend(constraints)
 
-  resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)
-
-  if options.cache_dir:
-    resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs)
-  else:
-    resolver = Resolver(**resolver_kwargs)
-
   with TRACER.timed('Resolving distributions'):
     try:
-      resolveds = resolver.resolve(resolvables)
+      resolveds = resolve_multi(resolvables,
+                                interpreters=interpreters,
+                                platforms=options.platform,
+                                cache=options.cache_dir,
+                                cache_ttl=options.cache_ttl)
+
+      for dist in resolveds:
+        log('  %s' % dist, v=options.verbosity)
+        pex_builder.add_distribution(dist)
+        pex_builder.add_requirement(dist.as_requirement())
     except Unsatisfiable as e:
       die(e)
 
-  for dist in resolveds:
-    log('  %s' % dist, v=options.verbosity)
-    pex_builder.add_distribution(dist)
-    pex_builder.add_requirement(dist.as_requirement())
-
   if options.entry_point and options.script:
     die('Must specify at most one entry point or script.', INVALID_OPTIONS)
 
@@ -585,8 +603,8 @@ def main(args=None):
       os.rename(tmp_name, options.pex_name)
       return 0
 
-    if options.platform != Platform.current():
-      log('WARNING: attempting to run PEX with differing platform!')
+    if options.platform and Platform.current() not in options.platform:
+      log('WARNING: attempting to run PEX with incompatible platforms!')
 
     pex_builder.freeze()
 

pex/resolver.py

@@ -291,17 +291,15 @@ def build(self, package, options):
     return DistributionHelper.distribution_from_path(target)
 
 
-def resolve(
-    requirements,
-    fetchers=None,
-    interpreter=None,
-    platform=None,
-    context=None,
-    precedence=None,
-    cache=None,
-    cache_ttl=None,
-    allow_prereleases=None):
-
+def resolve(requirements,
+            fetchers=None,
+            interpreter=None,
+            platform=None,
+            context=None,
+            precedence=None,
+            cache=None,
+            cache_ttl=None,
+            allow_prereleases=None):
   """Produce all distributions needed to (recursively) meet `requirements`
 
   :param requirements: An iterator of Requirement-like things, either
@@ -359,19 +357,85 @@ def resolve(
     classes.
   """
 
-  builder = ResolverOptionsBuilder(
-      fetchers=fetchers,
-      allow_prereleases=allow_prereleases,
-      precedence=precedence,
-      context=context,
-  )
+  builder = ResolverOptionsBuilder(fetchers=fetchers,
+                                   allow_prereleases=allow_prereleases,
+                                   precedence=precedence,
+                                   context=context)
 
   if cache:
-    resolver = CachingResolver(
-      cache, cache_ttl,
-      allow_prereleases=allow_prereleases, interpreter=interpreter, platform=platform)
+    resolver = CachingResolver(cache,
+                               cache_ttl,
+                               allow_prereleases=allow_prereleases,
+                               interpreter=interpreter,
+                               platform=platform)
   else:
-    resolver = Resolver(
-      allow_prereleases=allow_prereleases, interpreter=interpreter, platform=platform)
+    resolver = Resolver(allow_prereleases=allow_prereleases,
+                        interpreter=interpreter,
+                        platform=platform)
 
   return resolver.resolve(resolvables_from_iterable(requirements, builder))
+
+
+def resolve_multi(requirements,
+                  fetchers=None,
+                  interpreters=None,
+                  platforms=None,
+                  context=None,
+                  precedence=None,
+                  cache=None,
+                  cache_ttl=None,
+                  allow_prereleases=None):
+  """A generator function that produces all distributions needed to meet `requirements`
+  for multiple interpreters and/or platforms.
+
+  :param requirements: An iterator of Requirement-like things, either
+    :class:`pkg_resources.Requirement` objects or requirement strings.
+  :keyword fetchers: (optional) A list of :class:`Fetcher` objects for locating packages.  If
+    unspecified, the default is to look for packages on PyPI.
+  :keyword interpreters: (optional) An iterable of :class:`PythonInterpreter` objects to use
+    for building distributions and for testing distribution compatibility.
+  :keyword platforms: (optional) An iterable of PEP425-compatible platform strings to use for
+    filtering compatible distributions.  If unspecified, the current platform is used, as
+    determined by `Platform.current()`.
+  :keyword context: (optional) A :class:`Context` object to use for network access.  If
+    unspecified, the resolver will attempt to use the best available network context.
+  :keyword precedence: (optional) An ordered list of allowable :class:`Package` classes
+    to be used for producing distributions.  For example, if precedence is supplied as
+    ``(WheelPackage, SourcePackage)``, wheels will be preferred over building from source, and
+    eggs will not be used at all.  If ``(WheelPackage, EggPackage)`` is suppplied, both wheels and
+    eggs will be used, but the resolver will not resort to building anything from source.
+  :keyword cache: (optional) A directory to use to cache distributions locally.
+  :keyword cache_ttl: (optional integer in seconds) If specified, consider non-exact matches when
+    resolving requirements.  For example, if ``setuptools==2.2`` is specified and setuptools 2.2 is
+    available in the cache, it will always be used.  However, if a non-exact requirement such as
+    ``setuptools>=2,<3`` is specified and there exists a setuptools distribution newer than
+    cache_ttl seconds that satisfies the requirement, then it will be used.  If the distribution
+    is older than cache_ttl seconds, it will be ignored.  If ``cache_ttl`` is not specified,
+    resolving inexact requirements will always result in making network calls through the
+    ``context``.
+  :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
+    unspecified only stable versions will be resolved, unless explicitly included.
+  :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``.
+  :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
+  :raises Untranslateable: If no compatible distributions could be acquired for
+    a particular requirement.
+  """
+
+  interpreters = interpreters or [PythonInterpreter.get()]
+  platforms = platforms or [Platform.current()]
+
+  seen = set()
+  for interpreter in interpreters:
+    for platform in platforms:
+      for resolvable in resolve(requirements,
+                                fetchers,
+                                interpreter,
+                                platform,
+                                context,
+                                precedence,
+                                cache,
+                                cache_ttl,
+                                allow_prereleases):
+        if resolvable not in seen:
+          seen.add(resolvable)
+          yield resolvable

pex/testing.py

@@ -46,6 +46,13 @@ def random_bytes(length):
       map(chr, (random.randint(ord('a'), ord('z')) for _ in range(length)))).encode('utf-8')
 
 
+def get_dep_dist_names_from_pex(pex_path, match_prefix=''):
+  """Given an on-disk pex, extract all of the unique first-level paths under `.deps`."""
+  with zipfile.ZipFile(pex_path) as pex_zip:
+    dep_gen = (f.split(os.sep)[1] for f in pex_zip.namelist() if f.startswith('.deps/'))
+    return set(item for item in dep_gen if item.startswith(match_prefix))
+
+
 @contextlib.contextmanager
 def temporary_content(content_map, interp=None, seed=31337):
   """Write content to disk where content is map from string => (int, string).

tests/test_integration.py

@@ -10,9 +10,20 @@
 
 from pex.compatibility import WINDOWS
 from pex.installer import EggInstaller
-from pex.testing import run_pex_command, run_simple_pex, run_simple_pex_test, temporary_content
+from pex.testing import (
+    get_dep_dist_names_from_pex,
+    run_pex_command,
+    run_simple_pex,
+    run_simple_pex_test,
+    temporary_content
+)
 from pex.util import DistributionHelper, named_temporary_file
 
+NOT_CPYTHON_36 = (
+  "hasattr(sys, 'pypy_version_info') or "
+  "(sys.version_info[0], sys.version_info[1]) != (3, 6)"
+)
+
 
 def test_pex_execute():
   body = "print('Hello')"
@@ -155,3 +166,23 @@ def do_something():
     so, rc = run_simple_pex_test('', env={'PEX_SCRIPT': 'my_app'}, dists=[dist])
     assert so.decode('utf-8').strip() == error_msg
     assert rc == 1
+
+
+@pytest.mark.skipif(NOT_CPYTHON_36)
+def test_pex_multi_resolve():
+  """Tests multi-interpreter + multi-platform resolution."""
+  with temporary_dir() as output_dir:
+    pex_path = os.path.join(output_dir, 'pex.pex')
+    results = run_pex_command(['--disable-cache',
+                               'lxml==3.8.0',
+                               '--platform=manylinux1-x86_64',
+                               '--platform=macosx-10.6-x86_64',
+                               '--python=python2.7',
+                               '--python=python3.6',
+                               '-o', pex_path])
+    results.assert_success()
+
+    included_dists = get_dep_dist_names_from_pex(pex_path, 'lxml')
+    assert len(included_dists) == 4
+    for dist_substr in ('-cp27-', '-cp36-', '-manylinux1_x86_64', '-macosx_'):
+      assert any(dist_substr in f for f in included_dists)