create a Package abstraction for twitter.common.python

This takes ExtendedLink and turns it into a Package abstraction.  This is
the biggest change necessary to move over to supporting both Eggs and Wheels
together.

High level code summaries:
  1. move most of http/link.py to package.py
     a. ExtendedLink -> Package
     b. SourceLink -> SourcePackage
     c. EggLink -> EggPackage
     d. a lot of s/link/package/ (accounts for most of the minor line changes.)
  2. Package .satisfies and .compatible are two separate concepts now (one is
     requirement specific, the other is platform/py version specific)
  3. Fixed Obtainer abstraction leakage and generic-ified 'package precedence'
     (this is similar to how pkg_resources does it.)
  4. Moves Platform.distribution_compatible to package.py since that's where the
     concept belongs

The only remaining CL is adding WheelPackage and it's very minor (~100 line
diff on top of this.) NOTE: this is rebased off the merge of r/86 + r/87 +
r/91.

Testing Done:
git clean -fdx && ./pants.bootstrap tests/python/twitter/pants:all -v

Reviewed at https://rbcommons.com/s/twitter/r/92/

(sapling split of e606f443b99ea401e1b3654e4f689b69f61e56a6)

src/python/twitter/common/python/environment.py

@@ -1,13 +1,15 @@
 from __future__ import absolute_import, print_function
 
 import os
+import site
 import sys
 import uuid
 
 from .common import open_zip, safe_mkdir, safe_rmtree
+from .interpreter import PythonInterpreter
+from .package import distribution_compatible
 from .pex_builder import PEXBuilder
 from .pex_info import PexInfo
-from .platforms import Platform
 from .tracer import Tracer
 from .util import CacheHelper, DistributionHelper
 
@@ -65,8 +67,10 @@ def write_zipped_internal_cache(cls, pex, pex_info):
     prefix_length = len(pex_info.internal_cache) + 1
     distributions = []
     with open_zip(pex) as zf:
+      # Distribution names are the first element after ".deps/" and before the next "/"
       distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
           for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
+      # Create Distribution objects from these, and possibly write to disk if necessary.
       for distribution_name in distribution_names:
         internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
         dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
@@ -75,7 +79,8 @@ def write_zipped_internal_cache(cls, pex, pex_info):
           continue
         dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
             zf, internal_dist_path)
-        target_dir = os.path.join(pex_info.install_cache, '%s.%s' % (distribution_name, dist_digest))
+        target_dir = os.path.join(pex_info.install_cache, '%s.%s' % (
+            distribution_name, dist_digest))
         with TRACER.timed('Caching %s into %s' % (dist, target_dir)):
           distributions.append(CacheHelper.cache_distribution(zf, internal_dist_path, target_dir))
     return distributions
@@ -92,14 +97,15 @@ def load_internal_cache(cls, pex, pex_info):
         for dist in cls.write_zipped_internal_cache(pex, pex_info):
           yield dist
 
-  def __init__(self, pex, pex_info, platform=Platform.current(), python=Platform.python()):
+  def __init__(self, pex, pex_info, interpreter=None, **kw):
     self._internal_cache = os.path.join(pex, pex_info.internal_cache)
     self._pex = pex
     self._pex_info = pex_info
     self._activated = False
     self._working_set = None
-    super(PEXEnvironment, self).__init__(platform=platform, python=python,
-        search_path=sys.path if pex_info.inherit_path else [])
+    self._interpreter = interpreter or PythonInterpreter.get()
+    super(PEXEnvironment, self).__init__(
+        search_path=sys.path if pex_info.inherit_path else [], **kw)
 
   def update_candidate_distributions(self, distribution_iter):
     for dist in distribution_iter:
@@ -108,7 +114,7 @@ def update_candidate_distributions(self, distribution_iter):
           self.add(dist)
 
   def can_add(self, dist):
-    return Platform.distribution_compatible(dist, self.python, self.platform)
+    return distribution_compatible(dist, self._interpreter, self.platform)
 
   def activate(self):
     if not self._activated:
@@ -130,7 +136,6 @@ def _activate(self):
 
     working_set = WorkingSet([])
 
-    # for req in all_reqs:
     with TRACER.timed('Resolving %s' %
         ' '.join(map(str, all_reqs)) if all_reqs else 'empty dependency list'):
       try:
@@ -143,8 +148,12 @@ def _activate(self):
         raise
 
     for dist in resolved:
-      with TRACER.timed('Activated %s' % dist):
+      with TRACER.timed('Activating %s' % dist):
         working_set.add(dist)
         dist.activate()
 
+        if os.path.isdir(dist.location):
+          with TRACER.timed('Adding sitedir'):
+            site.addsitedir(dist.location)
+
     return working_set

src/python/twitter/common/python/http/__init__.py

@@ -1,13 +1,9 @@
 from .crawler import Crawler
 from .http import CachedWeb, FetchError, Web
-from .link import EggLink, Link, SourceLink
 
 __all__ = (
   CachedWeb,
   Crawler,
-  EggLink,
   FetchError,
-  Link,
-  SourceLink,
   Web,
 )

src/python/twitter/common/python/http/crawler.py

@@ -10,11 +10,9 @@
 
 if PY3:
   from queue import Empty, Queue
-  import urllib.error as urllib_error
   from urllib.parse import urlparse, urljoin
 else:
   from Queue import Empty, Queue
-  import urllib2 as urllib_error
   from urlparse import urlparse, urljoin
 
 

src/python/twitter/common/python/http/link.py

@@ -3,21 +3,10 @@
 import contextlib
 import os
 import posixpath
-import tarfile
-import zipfile
 
-from .http import FetchError
-
-from ..base import maybe_requirement
 from ..common import safe_mkdir, safe_mkdtemp
 from ..compatibility import PY3
-
-from pkg_resources import (
-    Distribution,
-    EGG_NAME,
-    parse_version,
-    safe_name,
-)
+from .http import FetchError
 
 if PY3:
   import urllib.parse as urlparse
@@ -84,150 +73,3 @@ def fetch(self, location=None, conn_timeout=None):
     except (FetchError, IOError) as e:
       raise self.UnreadableLink('Failed to fetch %s to %s: %s' % (self.url, location, e))
     return target
-
-
-class ExtendedLink(Link):
-  @property
-  def name(self):
-    return NotImplementedError
-
-  @property
-  def version(self):
-    return parse_version(self.raw_version)
-
-  @property
-  def raw_version(self):
-    return NotImplementedError
-
-  @property
-  def py_version(self):
-    return None
-
-  @property
-  def platform(self):
-    return None
-
-  def satisfies(self, requirement):
-    """Does the signature of this filename match the requirement (pkg_resources.Requirement)?"""
-    requirement = maybe_requirement(requirement)
-    distribution = Distribution(project_name=self.name, version=self.raw_version,
-      py_version=self.py_version, platform=self.platform)
-    if distribution.key != requirement.key:
-      return False
-    return self.raw_version in requirement
-
-
-class SourceLink(ExtendedLink):
-  """A Target providing source that can be built into a Distribution via Installer."""
-
-  EXTENSIONS = {
-    '.tar': (tarfile.TarFile.open, tarfile.ReadError),
-    '.tar.gz': (tarfile.TarFile.open, tarfile.ReadError),
-    '.tar.bz2': (tarfile.TarFile.open, tarfile.ReadError),
-    '.tgz': (tarfile.TarFile.open, tarfile.ReadError),
-    '.zip': (zipfile.ZipFile, zipfile.BadZipfile)
-  }
-
-  @classmethod
-  def split_fragment(cls, fragment):
-    """heuristic to split by version name/fragment:
-
-       >>> split_fragment('pysolr-2.1.0-beta')
-       ('pysolr', '2.1.0-beta')
-       >>> split_fragment('cElementTree-1.0.5-20051216')
-       ('cElementTree', '1.0.5-20051216')
-       >>> split_fragment('pil-1.1.7b1-20090412')
-       ('pil', '1.1.7b1-20090412')
-       >>> split_fragment('django-plugin-2-2.3')
-       ('django-plugin-2', '2.3')
-    """
-    def likely_version_component(enumerated_fragment):
-      return sum(bool(v and v[0].isdigit()) for v in enumerated_fragment[1].split('.'))
-    fragments = fragment.split('-')
-    if len(fragments) == 1:
-      return fragment, ''
-    max_index, _ = max(enumerate(fragments), key=likely_version_component)
-    return '-'.join(fragments[0:max_index]), '-'.join(fragments[max_index:])
-
-  def __init__(self, url, **kw):
-    super(SourceLink, self).__init__(url, **kw)
-
-    for ext, class_info in self.EXTENSIONS.items():
-      if self.filename.endswith(ext):
-        self._archive_class = class_info
-        fragment = self.filename[:-len(ext)]
-        break
-    else:
-      raise self.InvalidLink('%s does not end with any of: %s' % (
-          self.filename, ' '.join(self.EXTENSIONS)))
-    self._name, self._raw_version = self.split_fragment(fragment)
-
-  @property
-  def name(self):
-    return safe_name(self._name)
-
-  @property
-  def raw_version(self):
-    return safe_name(self._raw_version)
-
-  @staticmethod
-  def first_nontrivial_dir(path):
-    files = os.listdir(path)
-    if len(files) == 1 and os.path.isdir(os.path.join(path, files[0])):
-      return SourceLink.first_nontrivial_dir(os.path.join(path, files[0]))
-    else:
-      return path
-
-  def _unpack(self, filename, location=None):
-    """Unpack this source target into the path if supplied.  If the path is not supplied, a
-       temporary directory will be created."""
-    path = location or safe_mkdtemp()
-    archive_class, error_class = self._archive_class
-    try:
-      with contextlib.closing(archive_class(filename)) as package:
-        package.extractall(path=path)
-    except error_class:
-      raise self.UnreadableLink('Could not read %s' % self.url)
-    return self.first_nontrivial_dir(path)
-
-  def fetch(self, location=None, conn_timeout=None):
-    target = super(SourceLink, self).fetch(conn_timeout=conn_timeout)
-    return self._unpack(target, location)
-
-
-class EggLink(ExtendedLink):
-  """A Target providing an egg."""
-
-  def __init__(self, url, **kw):
-    super(EggLink, self).__init__(url, **kw)
-    filename, ext = os.path.splitext(self.filename)
-    if ext.lower() != '.egg':
-      raise self.InvalidLink('Not an egg: %s' % filename)
-    matcher = EGG_NAME(filename)
-    if not matcher:
-      raise self.InvalidLink('Could not match egg: %s' % filename)
-
-    self._name, self._raw_version, self._py_version, self._platform = matcher.group(
-        'name', 'ver', 'pyver', 'plat')
-
-    if self._raw_version is None or self._py_version is None:
-      raise self.InvalidLink('url with .egg extension but bad name: %s' % url)
-
-  def __hash__(self):
-    return hash((self.name, self.version, self.py_version, self.platform))
-
-  @property
-  def name(self):
-    return safe_name(self._name)
-
-  @property
-  def raw_version(self):
-    return safe_name(self._raw_version)
-
-  @property
-  def py_version(self):
-    return self._py_version
-
-  @property
-  def platform(self):
-    return self._platform

src/python/twitter/common/python/installer.py

@@ -122,9 +122,11 @@ class Installer(InstallerBase):
     Install an unpacked distribution with a setup.py.
 
     Simple example:
-      >>> from twitter.common.python.http import Web, SourceLink
-      >>> tornado_tgz = SourceLink('http://pypi.python.org/packages/source/t/tornado/tornado-2.3.tar.gz',
-      ...                          opener=Web())
+      >>> from twitter.common.python.package import SourcePackage
+      >>> from twitter.common.python.http import Web
+      >>> tornado_tgz = SourcePackage(
+      ...    'http://pypi.python.org/packages/source/t/tornado/tornado-2.3.tar.gz',
+      ...    opener=Web())
       >>> tornado_installer = Installer(tornado_tgz.fetch())
       >>> tornado_installer.distribution()
       tornado 2.3 (/private/var/folders/Uh/UhXpeRIeFfGF7HoogOKC+++++TI/-Tmp-/tmpLLe_Ph/lib/python2.6/site-packages)

src/python/twitter/common/python/interpreter.py

@@ -161,6 +161,12 @@ def hashbang(self):
     }
     return '#!/usr/bin/env %s' % hashbang_string
 
+  @property
+  def python(self):
+    # return the python version in the format of the 'python' key for distributions
+    # specifically, '2.6', '2.7', '3.2', etc.
+    return '%d.%d' % (self.version[0:2])
+
   def __str__(self):
     return '%s-%s.%s.%s' % (self._interpreter,
       self._version[0], self._version[1], self._version[2])
@@ -351,8 +357,7 @@ def replace(cls, requirement):
         break
     else:
       raise cls.InterpreterNotFound('Could not find interpreter matching filter!')
-    cls.sanitize_environment()
-    os.execv(pi.binary, [pi.binary] + sys.argv)
+    os.execve(pi.binary, [pi.binary] + sys.argv, cls.sanitized_environment())
 
   def __init__(self, binary, identity, extras=None):
     """Construct a PythonInterpreter.
@@ -364,7 +369,7 @@ def __init__(self, binary, identity, extras=None):
        :param extras: A mapping from (dist.key, dist.version) to dist.location
                       of the extras associated with this interpreter.
     """
-    self._binary = binary
+    self._binary = os.path.realpath(binary)
     self._binary_stat = os.stat(self._binary)
     self._extras = extras or {}
     self._identity = identity
@@ -388,9 +393,7 @@ def identity(self):
 
   @property
   def python(self):
-    # return the python version in the format of the 'python' key for distributions
-    # specifically, '2.6', '2.7', '3.2', etc.
-    return '%d.%d' % (self._identity.version[0:2])
+    return self._identity.python
 
   @property
   def version(self):
@@ -416,12 +419,12 @@ def __hash__(self):
     return hash(self._binary_stat)
 
   def __eq__(self, other):
-    if not isinstance(other, self.__class__):
+    if not isinstance(other, PythonInterpreter):
       return False
     return self._binary_stat == other._binary_stat
 
   def __lt__(self, other):
-    if not isinstance(other, self.__class__):
+    if not isinstance(other, PythonInterpreter):
       return False
     return self.version < other.version