Adds an interpreter cache to the pex tool. Fixes pex-tool#47 and pex-…

…tool#48.

* Adds an interpreter cache to the pex tool and self-resolves dependencies
  (wheel, setuptools) should they be unavailable at runtime.  This fixes the
  issue where "pex" would fail to run in a fresh virtualenv without doing
  "pip install pex[whl]".

* Fixes test breakage with requests >= 2.5.2

CHANGES.rst

@@ -3,9 +3,22 @@ CHANGES
 =======
 
 ----------
-0.8.6
+0.8.7-dev0
 ----------
 
+* Bug fix: Make ``pip install pex`` work better by removing ``extras_requires`` on the
+  ``console_script`` entry point.  Fixes `#48 <https://github.com/pantsbuild/pex/issues/48>`_
+
+* New feature: Adds an interpreter cache to the ``pex`` tool.  If the user does not explicitly
+  disable the wheel feature and attempts to build a pex with wheels but does not have the wheel
+  package installed, pex will download it in order to make the feature work.
+  Implements `#47 <https://github.com/pantsbuild/pex/issues/47>`_ in order to
+  fix `#48 <https://github.com/pantsbuild/pex/issues/48>`_
+
+-----
+0.8.6
+-----
+
 * Bug fix: Honor installed sys.excepthook in pex teardown.
   `RB #1733 <https://rbcommons.com/s/twitter/r/1733>`_
 

pex/bin/pex.py

@@ -8,25 +8,32 @@
 
 from __future__ import absolute_import, print_function
 
+import functools
 import os
 import shutil
 import sys
 from optparse import OptionParser
 
-from pex.common import safe_delete, safe_mkdtemp
+from pex.archiver import Archiver
+from pex.base import maybe_requirement
+from pex.common import safe_delete, safe_mkdir, safe_mkdtemp
+from pex.crawler import Crawler
 from pex.fetcher import Fetcher, PyPIFetcher
+from pex.http import Context
 from pex.installer import EggInstaller, Packager, WheelInstaller
 from pex.interpreter import PythonInterpreter
+from pex.iterator import Iterator
 from pex.package import EggPackage, Package, SourcePackage, WheelPackage
 from pex.pex import PEX
 from pex.pex_builder import PEXBuilder
 from pex.platforms import Platform
 from pex.resolver import resolve as requirement_resolver
 from pex.tracer import TRACER, TraceLogger
 from pex.translator import ChainedTranslator, EggTranslator, SourceTranslator, WheelTranslator
-from pex.version import __version__
+from pex.version import __setuptools_requirement, __version__, __wheel_requirement
 
 CANNOT_DISTILL = 101
+CANNOT_SETUP_INTERPRETER = 102
 
 
 def die(msg, error_code=1):
@@ -129,12 +136,19 @@ def configure_clp():
       help='Inherit the contents of sys.path (including site-packages) running the pex. '
            '[Default: %default]')
 
+  parser.add_option(
+      '--interpreter-cache-dir',
+      dest='interpreter_cache_dir',
+      default=os.path.expanduser('~/.pex/interpreters'),
+      help='The interpreter cache to use for keeping track of interpreter dependencies '
+           'for the pex tool. [Default: %default]')
+
   parser.add_option(
       '--cache-dir',
       dest='cache_dir',
       default=os.path.expanduser('~/.pex/build'),
       help='The local cache directory to use for speeding up requirement '
-           'lookups; [Default: %default]')
+           'lookups. [Default: %default]')
 
   parser.add_option(
       '--cache-ttl',
@@ -202,8 +216,87 @@ def configure_clp():
   return parser
 
 
+def _safe_link(src, dst):
+  try:
+    os.unlink(dst)
+  except OSError:
+    pass
+  os.symlink(src, dst)
+
+
+def _resolve_and_link_interpreter(requirement, fetchers, target_link, installer_provider):
+  # Short-circuit if there is a local copy
+  if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
+    egg = EggPackage(os.path.realpath(target_link))
+    if egg.satisfies(requirement):
+      return egg
+
+  context = Context.get()
+  iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
+  links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)]
+
+  with TRACER.timed('Interpreter cache resolving %s' % requirement, V=2):
+    for link in links:
+      with TRACER.timed('Fetching %s' % link, V=3):
+        sdist = context.fetch(link)
+
+      with TRACER.timed('Installing %s' % link, V=3):
+        installer = installer_provider(sdist)
+        dist_location = installer.bdist()
+        target_location = os.path.join(
+            os.path.dirname(target_link), os.path.basename(dist_location))
+        shutil.move(dist_location, target_location)
+        _safe_link(target_location, target_link)
+
+      return EggPackage(target_location)
+
+
+def resolve_interpreter(cache, fetchers, interpreter, requirement):
+  """Resolve an interpreter with a specific requirement.
+
+     Given a :class:`PythonInterpreter` and a requirement, return an
+     interpreter with the capability of resolving that requirement or
+     ``None`` if it's not possible to install a suitable requirement."""
+  requirement = maybe_requirement(requirement)
+
+  # short circuit
+  if interpreter.satisfies([requirement]):
+    return interpreter
+
+  def installer_provider(sdist):
+    return EggInstaller(
+        Archiver.unpack(sdist),
+        strict=requirement.key != 'setuptools',
+        interpreter=interpreter)
+
+  interpreter_dir = os.path.join(cache, str(interpreter.identity))
+  safe_mkdir(interpreter_dir)
+
+  egg = _resolve_and_link_interpreter(
+      requirement,
+      fetchers,
+      os.path.join(interpreter_dir, requirement.key),
+      installer_provider)
+
+  if egg:
+    return interpreter.with_extra(egg.name, egg.raw_version, egg.path)
+
+
+def fetchers_from_options(options):
+  fetchers = [Fetcher(options.repos)]
+
+  if options.pypi:
+    fetchers.append(PyPIFetcher())
+
+  if options.indices:
+    fetchers.extend(PyPIFetcher(index) for index in options.indices)
+
+  return fetchers
+
+
 def interpreter_from_options(options):
   interpreter = None
+
   if options.python:
     if os.path.exists(options.python):
       interpreter = PythonInterpreter.from_binary(options.python)
@@ -213,11 +306,23 @@ def interpreter_from_options(options):
       die('Failed to find interpreter: %s' % options.python)
   else:
     interpreter = PythonInterpreter.get()
-  return interpreter
 
+  with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
+    fetchers = fetchers_from_options(options)
+
+    resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, fetchers)
+
+    # resolve setuptools
+    interpreter = resolve(interpreter, __setuptools_requirement)
+
+    # possibly resolve wheel
+    if interpreter and options.use_wheel:
+      interpreter = resolve(interpreter, __wheel_requirement)
+
+    return interpreter
 
-def translator_from_options(options):
-  interpreter = interpreter_from_options(options)
+
+def translator_from_options(interpreter, options):
   platform = options.platform
 
   translators = []
@@ -237,7 +342,11 @@ def translator_from_options(options):
 
 
 def build_pex(args, options):
-  interpreter = interpreter_from_options(options)
+  with TRACER.timed('Resolving interpreter', V=2):
+    interpreter = interpreter_from_options(options)
+
+  if interpreter is None:
+    die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
 
   pex_builder = PEXBuilder(
       path=safe_mkdtemp(),
@@ -253,17 +362,8 @@ def build_pex(args, options):
 
   installer = WheelInstaller if options.use_wheel else EggInstaller
 
-  interpreter = interpreter_from_options(options)
-
-  fetchers = [Fetcher(options.repos)]
-
-  if options.pypi:
-    fetchers.append(PyPIFetcher())
-
-  if options.indices:
-    fetchers.extend(PyPIFetcher(index) for index in options.indices)
-
-  translator = translator_from_options(options)
+  fetchers = fetchers_from_options(options)
+  translator = translator_from_options(interpreter, options)
 
   if options.use_wheel:
     precedence = (WheelPackage, EggPackage, SourcePackage)
@@ -322,7 +422,8 @@ def main():
 
   with TraceLogger.env_override(PEX_VERBOSE=options.verbosity):
 
-    pex_builder = build_pex(args, options)
+    with TRACER.timed('Building pex'):
+      pex_builder = build_pex(args, options)
 
     if options.pex_name is not None:
       log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)

pex/http.py

@@ -58,7 +58,9 @@ def register(cls, context_impl):
   def get(cls):
     for context_class in cls._REGISTRY:
       try:
-        return context_class()
+        context = context_class()
+        TRACER.log('Constructed %s context %r' % (context.__class__.__name__, context), V=4)
+        return context
       except cls.Error:
         continue
     raise cls.Error('Could not initialize a request context.')
@@ -121,6 +123,11 @@ def content(self, link):
       encoding = message_from_string(str(fp.headers)).get_content_charset(self.DEFAULT_ENCODING)
       return fp.read().decode(encoding, 'replace')
 
+  def __init__(self, *args, **kw):
+    TRACER.log('Warning, using a UrllibContext which is known to be flaky.')
+    TRACER.log('Please build pex with the requests module for more reliable downloads.')
+    super(UrllibContext, self).__init__(*args, **kw)
+
 
 Context.register(UrllibContext)
 
@@ -185,6 +192,9 @@ def _create_session(max_retries):
     return session
 
   def __init__(self, session=None, verify=True, max_retries=5):
+    if requests is None:
+      raise RuntimeError('requests is not available.  Cannot use a RequestsContext.')
+
     self._verify = verify
 
     max_retries = int(os.environ.get('PEX_HTTP_RETRIES', max_retries))

pex/version.py

@@ -1 +1,4 @@
 __version__ = '0.8.7-dev0'
+
+__setuptools_requirement = 'setuptools>=2.2,<8'
+__wheel_requirement = 'wheel>=0.24.0,<0.25.0'

setup.py

@@ -11,6 +11,11 @@
 
 
 # This seems to be a fairly standard version file pattern.
+#
+# Populates the following variables:
+#   __version__
+#   __setuptools_requirement
+#   __wheel_requirement
 __version__ = ''
 version_py_file = os.path.join(os.path.dirname(__file__), 'pex', 'version.py')
 with open(version_py_file) as version_py:
@@ -36,7 +41,7 @@
     'pex.bin',
   ],
   install_requires = [
-    'setuptools>=2.2,<8',
+    __setuptools_requirement,
   ],
   tests_require = [
     'mock',
@@ -48,10 +53,7 @@
   ],
   entry_points = {
     'console_scripts': [
-      'pex = pex.bin.pex:main [whl]',
+      'pex = pex.bin.pex:main',
     ],
   },
-  extras_require = {
-    'whl': ['wheel>=0.24.0,<0.25.0'],
-  },
 )

tests/test_http.py

@@ -5,6 +5,7 @@
 import pytest
 from twitter.common.contextutil import temporary_file
 
+from pex.compatibility import PY2
 from pex.http import Context, RequestsContext, StreamFilelike, UrllibContext
 from pex.link import Link
 
@@ -27,6 +28,12 @@
 NO_REQUESTS = 'RequestsMock is None or requests is None'
 
 
+try:
+  from httplib import HTTPMessage
+except ImportError:
+  from http.client import HTTPMessage
+
+
 def make_md5(blob):
   md5 = hashlib.md5()
   md5.update(blob)
@@ -108,10 +115,14 @@ def __init__(self, data):
     self.status = 200
     self.version = 'HTTP/1.1'
     self.reason = 'OK'
-    self.msg = mock.Mock()
+    if PY2:
+      self.msg = HTTPMessage(BytesIO('Content-Type: application/x-compressed\r\n'))
+    else:
+      self.msg = HTTPMessage()
+      self.msg.add_header('Content-Type', 'application/x-compressed')
 
   def getheaders(self):
-    return [('Content-Type', 'application/x-compressed')]
+    return list(self.msg.items())
 
   def isclosed(self):
     return self.closed