From 71bb9c367d0a333e7859fc7be9c8b31de8069cdb Mon Sep 17 00:00:00 2001
From: "Brian Wickman (Twitter)" <wickman@apache.org>
Date: Tue, 18 Mar 2014 11:52:07 -0700
Subject: [PATCH] create a Package abstraction for twitter.common.python

This takes ExtendedLink and turns it into a Package abstraction.  This is
the biggest change necessary to move over to supporting both Eggs and Wheels
together.

High level code summaries:
  1. move most of http/link.py to package.py
     a. ExtendedLink -> Package
     b. SourceLink -> SourcePackage
     c. EggLink -> EggPackage
     d. a lot of s/link/package/ (accounts for most of the minor line changes.)
  2. Package .satisfies and .compatible are two separate concepts now (one is
     requirement specific, the other is platform/py version specific)
  3. Fixed Obtainer abstraction leakage and generic-ified 'package precedence'
     (this is similar to how pkg_resources does it.)
  4. Moves Platform.distribution_compatible to package.py since that's where the
     concept belongs

The only remaining CL is adding WheelPackage and it's very minor (~100 line
diff on top of this.) NOTE: this is rebased off the merge of r/86 + r/87 +
r/91.

Testing Done:
git clean -fdx && ./pants.bootstrap tests/python/twitter/pants:all -v

Reviewed at https://rbcommons.com/s/twitter/r/92/

(sapling split of e606f443b99ea401e1b3654e4f689b69f61e56a6)
---
 .../twitter/common/python/environment.py      |  25 +-
 .../twitter/common/python/http/__init__.py    |   4 -
 .../twitter/common/python/http/crawler.py     |   2 -
 src/python/twitter/common/python/http/link.py | 160 +-----------
 src/python/twitter/common/python/installer.py |   8 +-
 .../twitter/common/python/interpreter.py      |  19 +-
 src/python/twitter/common/python/obtainer.py  |  59 +++--
 src/python/twitter/common/python/package.py   | 239 ++++++++++++++++++
 .../twitter/common/python/pex_bootstrapper.py |   2 +
 .../twitter/common/python/pex_builder.py      |  51 ++--
 src/python/twitter/common/python/platforms.py |   8 -
 src/python/twitter/common/python/resolver.py  |  28 +-
 .../twitter/common/python/translator.py       | 108 ++++----
 src/python/twitter/common/python/util.py      |  49 ++--
 tests/python/twitter/common/python/BUILD      |  20 +-
 tests/python/twitter/common/python/http/BUILD |  17 +-
 .../twitter/common/python/test_obtainer.py    |  51 ++--
 .../{http/test_link.py => test_package.py}    |  40 ++-
 .../python/twitter/common/python/test_util.py |  10 +-
 19 files changed, 514 insertions(+), 386 deletions(-)
 create mode 100644 src/python/twitter/common/python/package.py
 rename tests/python/twitter/common/python/{http/test_link.py => test_package.py} (74%)

diff --git a/src/python/twitter/common/python/environment.py b/src/python/twitter/common/python/environment.py
index 10ca0f919..50ba039c5 100644
--- a/src/python/twitter/common/python/environment.py
+++ b/src/python/twitter/common/python/environment.py
@@ -1,13 +1,15 @@
 from __future__ import absolute_import, print_function
 
 import os
+import site
 import sys
 import uuid
 
 from .common import open_zip, safe_mkdir, safe_rmtree
+from .interpreter import PythonInterpreter
+from .package import distribution_compatible
 from .pex_builder import PEXBuilder
 from .pex_info import PexInfo
-from .platforms import Platform
 from .tracer import Tracer
 from .util import CacheHelper, DistributionHelper
 
@@ -65,8 +67,10 @@ def write_zipped_internal_cache(cls, pex, pex_info):
     prefix_length = len(pex_info.internal_cache) + 1
     distributions = []
     with open_zip(pex) as zf:
+      # Distribution names are the first element after ".deps/" and before the next "/"
       distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
           for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
+      # Create Distribution objects from these, and possibly write to disk if necessary.
       for distribution_name in distribution_names:
         internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
         dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
@@ -75,7 +79,8 @@ def write_zipped_internal_cache(cls, pex, pex_info):
           continue
         dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
             zf, internal_dist_path)
-        target_dir = os.path.join(pex_info.install_cache, '%s.%s' % (distribution_name, dist_digest))
+        target_dir = os.path.join(pex_info.install_cache, '%s.%s' % (
+            distribution_name, dist_digest))
         with TRACER.timed('Caching %s into %s' % (dist, target_dir)):
           distributions.append(CacheHelper.cache_distribution(zf, internal_dist_path, target_dir))
     return distributions
@@ -92,14 +97,15 @@ def load_internal_cache(cls, pex, pex_info):
         for dist in cls.write_zipped_internal_cache(pex, pex_info):
           yield dist
 
-  def __init__(self, pex, pex_info, platform=Platform.current(), python=Platform.python()):
+  def __init__(self, pex, pex_info, interpreter=None, **kw):
     self._internal_cache = os.path.join(pex, pex_info.internal_cache)
     self._pex = pex
     self._pex_info = pex_info
     self._activated = False
     self._working_set = None
-    super(PEXEnvironment, self).__init__(platform=platform, python=python,
-        search_path=sys.path if pex_info.inherit_path else [])
+    self._interpreter = interpreter or PythonInterpreter.get()
+    super(PEXEnvironment, self).__init__(
+        search_path=sys.path if pex_info.inherit_path else [], **kw)
 
   def update_candidate_distributions(self, distribution_iter):
     for dist in distribution_iter:
@@ -108,7 +114,7 @@ def update_candidate_distributions(self, distribution_iter):
           self.add(dist)
 
   def can_add(self, dist):
-    return Platform.distribution_compatible(dist, self.python, self.platform)
+    return distribution_compatible(dist, self._interpreter, self.platform)
 
   def activate(self):
     if not self._activated:
@@ -130,7 +136,6 @@ def _activate(self):
 
     working_set = WorkingSet([])
 
-    # for req in all_reqs:
     with TRACER.timed('Resolving %s' %
         ' '.join(map(str, all_reqs)) if all_reqs else 'empty dependency list'):
       try:
@@ -143,8 +148,12 @@ def _activate(self):
         raise
 
     for dist in resolved:
-      with TRACER.timed('Activated %s' % dist):
+      with TRACER.timed('Activating %s' % dist):
         working_set.add(dist)
         dist.activate()
 
+        if os.path.isdir(dist.location):
+          with TRACER.timed('Adding sitedir'):
+            site.addsitedir(dist.location)
+
     return working_set
diff --git a/src/python/twitter/common/python/http/__init__.py b/src/python/twitter/common/python/http/__init__.py
index 607338d3b..1f7da2afe 100644
--- a/src/python/twitter/common/python/http/__init__.py
+++ b/src/python/twitter/common/python/http/__init__.py
@@ -1,13 +1,9 @@
 from .crawler import Crawler
 from .http import CachedWeb, FetchError, Web
-from .link import EggLink, Link, SourceLink
 
 __all__ = (
   CachedWeb,
   Crawler,
-  EggLink,
   FetchError,
-  Link,
-  SourceLink,
   Web,
 )
diff --git a/src/python/twitter/common/python/http/crawler.py b/src/python/twitter/common/python/http/crawler.py
index 75ea4aabe..23118965d 100644
--- a/src/python/twitter/common/python/http/crawler.py
+++ b/src/python/twitter/common/python/http/crawler.py
@@ -10,11 +10,9 @@
 
 if PY3:
   from queue import Empty, Queue
-  import urllib.error as urllib_error
   from urllib.parse import urlparse, urljoin
 else:
   from Queue import Empty, Queue
-  import urllib2 as urllib_error
   from urlparse import urlparse, urljoin
 
 
diff --git a/src/python/twitter/common/python/http/link.py b/src/python/twitter/common/python/http/link.py
index 7a84012f1..a8bb8ba81 100644
--- a/src/python/twitter/common/python/http/link.py
+++ b/src/python/twitter/common/python/http/link.py
@@ -3,21 +3,10 @@
 import contextlib
 import os
 import posixpath
-import tarfile
-import zipfile
 
-from .http import FetchError
-
-from ..base import maybe_requirement
 from ..common import safe_mkdir, safe_mkdtemp
 from ..compatibility import PY3
-
-from pkg_resources import (
-    Distribution,
-    EGG_NAME,
-    parse_version,
-    safe_name,
-)
+from .http import FetchError
 
 if PY3:
   import urllib.parse as urlparse
@@ -84,150 +73,3 @@ def fetch(self, location=None, conn_timeout=None):
     except (FetchError, IOError) as e:
       raise self.UnreadableLink('Failed to fetch %s to %s: %s' % (self.url, location, e))
     return target
-
-
-class ExtendedLink(Link):
-  @property
-  def name(self):
-    return NotImplementedError
-
-  @property
-  def version(self):
-    return parse_version(self.raw_version)
-
-  @property
-  def raw_version(self):
-    return NotImplementedError
-
-  @property
-  def py_version(self):
-    return None
-
-  @property
-  def platform(self):
-    return None
-
-  def satisfies(self, requirement):
-    """Does the signature of this filename match the requirement (pkg_resources.Requirement)?"""
-    requirement = maybe_requirement(requirement)
-    distribution = Distribution(project_name=self.name, version=self.raw_version,
-      py_version=self.py_version, platform=self.platform)
-    if distribution.key != requirement.key:
-      return False
-    return self.raw_version in requirement
-
-
-class SourceLink(ExtendedLink):
-  """A Target providing source that can be built into a Distribution via Installer."""
-
-  EXTENSIONS = {
-    '.tar': (tarfile.TarFile.open, tarfile.ReadError),
-    '.tar.gz': (tarfile.TarFile.open, tarfile.ReadError),
-    '.tar.bz2': (tarfile.TarFile.open, tarfile.ReadError),
-    '.tgz': (tarfile.TarFile.open, tarfile.ReadError),
-    '.zip': (zipfile.ZipFile, zipfile.BadZipfile)
-  }
-
-  @classmethod
-  def split_fragment(cls, fragment):
-    """heuristic to split by version name/fragment:
-
-       >>> split_fragment('pysolr-2.1.0-beta')
-       ('pysolr', '2.1.0-beta')
-       >>> split_fragment('cElementTree-1.0.5-20051216')
-       ('cElementTree', '1.0.5-20051216')
-       >>> split_fragment('pil-1.1.7b1-20090412')
-       ('pil', '1.1.7b1-20090412')
-       >>> split_fragment('django-plugin-2-2.3')
-       ('django-plugin-2', '2.3')
-    """
-    def likely_version_component(enumerated_fragment):
-      return sum(bool(v and v[0].isdigit()) for v in enumerated_fragment[1].split('.'))
-    fragments = fragment.split('-')
-    if len(fragments) == 1:
-      return fragment, ''
-    max_index, _ = max(enumerate(fragments), key=likely_version_component)
-    return '-'.join(fragments[0:max_index]), '-'.join(fragments[max_index:])
-
-  def __init__(self, url, **kw):
-    super(SourceLink, self).__init__(url, **kw)
-
-    for ext, class_info in self.EXTENSIONS.items():
-      if self.filename.endswith(ext):
-        self._archive_class = class_info
-        fragment = self.filename[:-len(ext)]
-        break
-    else:
-      raise self.InvalidLink('%s does not end with any of: %s' % (
-          self.filename, ' '.join(self.EXTENSIONS)))
-    self._name, self._raw_version = self.split_fragment(fragment)
-
-  @property
-  def name(self):
-    return safe_name(self._name)
-
-  @property
-  def raw_version(self):
-    return safe_name(self._raw_version)
-
-  @staticmethod
-  def first_nontrivial_dir(path):
-    files = os.listdir(path)
-    if len(files) == 1 and os.path.isdir(os.path.join(path, files[0])):
-      return SourceLink.first_nontrivial_dir(os.path.join(path, files[0]))
-    else:
-      return path
-
-  def _unpack(self, filename, location=None):
-    """Unpack this source target into the path if supplied.  If the path is not supplied, a
-       temporary directory will be created."""
-    path = location or safe_mkdtemp()
-    archive_class, error_class = self._archive_class
-    try:
-      with contextlib.closing(archive_class(filename)) as package:
-        package.extractall(path=path)
-    except error_class:
-      raise self.UnreadableLink('Could not read %s' % self.url)
-    return self.first_nontrivial_dir(path)
-
-  def fetch(self, location=None, conn_timeout=None):
-    target = super(SourceLink, self).fetch(conn_timeout=conn_timeout)
-    return self._unpack(target, location)
-
-
-class EggLink(ExtendedLink):
-  """A Target providing an egg."""
-
-  def __init__(self, url, **kw):
-    super(EggLink, self).__init__(url, **kw)
-    filename, ext = os.path.splitext(self.filename)
-    if ext.lower() != '.egg':
-      raise self.InvalidLink('Not an egg: %s' % filename)
-    matcher = EGG_NAME(filename)
-    if not matcher:
-      raise self.InvalidLink('Could not match egg: %s' % filename)
-
-    self._name, self._raw_version, self._py_version, self._platform = matcher.group(
-        'name', 'ver', 'pyver', 'plat')
-
-    if self._raw_version is None or self._py_version is None:
-      raise self.InvalidLink('url with .egg extension but bad name: %s' % url)
-
-  def __hash__(self):
-    return hash((self.name, self.version, self.py_version, self.platform))
-
-  @property
-  def name(self):
-    return safe_name(self._name)
-
-  @property
-  def raw_version(self):
-    return safe_name(self._raw_version)
-
-  @property
-  def py_version(self):
-    return self._py_version
-
-  @property
-  def platform(self):
-    return self._platform
diff --git a/src/python/twitter/common/python/installer.py b/src/python/twitter/common/python/installer.py
index b1fe43255..31543ab25 100644
--- a/src/python/twitter/common/python/installer.py
+++ b/src/python/twitter/common/python/installer.py
@@ -122,9 +122,11 @@ class Installer(InstallerBase):
     Install an unpacked distribution with a setup.py.
 
     Simple example:
-      >>> from twitter.common.python.http import Web, SourceLink
-      >>> tornado_tgz = SourceLink('http://pypi.python.org/packages/source/t/tornado/tornado-2.3.tar.gz',
-      ...                          opener=Web())
+      >>> from twitter.common.python.package import SourcePackage
+      >>> from twitter.common.python.http import Web
+      >>> tornado_tgz = SourcePackage(
+      ...    'http://pypi.python.org/packages/source/t/tornado/tornado-2.3.tar.gz',
+      ...    opener=Web())
       >>> tornado_installer = Installer(tornado_tgz.fetch())
       >>> tornado_installer.distribution()
       tornado 2.3 (/private/var/folders/Uh/UhXpeRIeFfGF7HoogOKC+++++TI/-Tmp-/tmpLLe_Ph/lib/python2.6/site-packages)
diff --git a/src/python/twitter/common/python/interpreter.py b/src/python/twitter/common/python/interpreter.py
index 30f9241ce..c2ee8162c 100644
--- a/src/python/twitter/common/python/interpreter.py
+++ b/src/python/twitter/common/python/interpreter.py
@@ -161,6 +161,12 @@ def hashbang(self):
     }
     return '#!/usr/bin/env %s' % hashbang_string
 
+  @property
+  def python(self):
+    # return the python version in the format of the 'python' key for distributions
+    # specifically, '2.6', '2.7', '3.2', etc.
+    return '%d.%d' % (self.version[0:2])
+
   def __str__(self):
     return '%s-%s.%s.%s' % (self._interpreter,
       self._version[0], self._version[1], self._version[2])
@@ -351,8 +357,7 @@ def replace(cls, requirement):
         break
     else:
       raise cls.InterpreterNotFound('Could not find interpreter matching filter!')
-    cls.sanitize_environment()
-    os.execv(pi.binary, [pi.binary] + sys.argv)
+    os.execve(pi.binary, [pi.binary] + sys.argv, cls.sanitized_environment())
 
   def __init__(self, binary, identity, extras=None):
     """Construct a PythonInterpreter.
@@ -364,7 +369,7 @@ def __init__(self, binary, identity, extras=None):
        :param extras: A mapping from (dist.key, dist.version) to dist.location
                       of the extras associated with this interpreter.
     """
-    self._binary = binary
+    self._binary = os.path.realpath(binary)
     self._binary_stat = os.stat(self._binary)
     self._extras = extras or {}
     self._identity = identity
@@ -388,9 +393,7 @@ def identity(self):
 
   @property
   def python(self):
-    # return the python version in the format of the 'python' key for distributions
-    # specifically, '2.6', '2.7', '3.2', etc.
-    return '%d.%d' % (self._identity.version[0:2])
+    return self._identity.python
 
   @property
   def version(self):
@@ -416,12 +419,12 @@ def __hash__(self):
     return hash(self._binary_stat)
 
   def __eq__(self, other):
-    if not isinstance(other, self.__class__):
+    if not isinstance(other, PythonInterpreter):
       return False
     return self._binary_stat == other._binary_stat
 
   def __lt__(self, other):
-    if not isinstance(other, self.__class__):
+    if not isinstance(other, PythonInterpreter):
       return False
     return self.version < other.version
 
diff --git a/src/python/twitter/common/python/obtainer.py b/src/python/twitter/common/python/obtainer.py
index acb414650..bbc8f1bda 100644
--- a/src/python/twitter/common/python/obtainer.py
+++ b/src/python/twitter/common/python/obtainer.py
@@ -1,6 +1,10 @@
 import itertools
 
-from .http import EggLink, SourceLink
+from .package import (
+     EggPackage,
+     Package,
+     SourcePackage,
+)
 from .tracer import TRACER
 from .translator import ChainedTranslator
 
@@ -11,7 +15,7 @@ class Obtainer(object):
 
     An Obtainer takes a Crawler, a list of Fetchers (which take requirements
     and tells us where to look for them) and a list of Translators (which
-    translate egg or source links into usable distributions) and turns them
+    translate egg or source packages into usable distributions) and turns them
     into a cohesive requirement pipeline.
 
     >>> from twitter.common.python.http import Crawler
@@ -25,7 +29,24 @@ class Obtainer(object):
     ...                                   'pygments', 'pylint', 'pytest'])
     >>> for d in distributions: d.activate()
   """
-  def __init__(self, crawler, fetchers, translators):
+  DEFAULT_PACKAGE_PRECEDENCE = (
+      EggPackage,
+      SourcePackage,
+  )
+
+  @classmethod
+  def package_type_precedence(cls, package, precedence=DEFAULT_PACKAGE_PRECEDENCE):
+    for rank, package_type in enumerate(reversed(precedence)):
+      if isinstance(package, package_type):
+        return rank
+    # If we do not recognize the package, it gets lowest precedence
+    return -1
+
+  @classmethod
+  def package_precedence(cls, package, precedence=DEFAULT_PACKAGE_PRECEDENCE):
+    return (package.version, cls.package_type_precedence(package, precedence=precedence))
+
+  def __init__(self, crawler, fetchers, translators, precedence=DEFAULT_PACKAGE_PRECEDENCE):
     self._crawler = crawler
     self._fetchers = fetchers
     # use maybe_list?
@@ -33,35 +54,27 @@ def __init__(self, crawler, fetchers, translators):
       self._translator = ChainedTranslator(*translators)
     else:
       self._translator = translators
+    self._precedence = precedence
 
   def translate_href(self, href):
-    for link_class in (EggLink, SourceLink):
-      try:
-        return link_class(href, opener=self._crawler.opener)
-      except link_class.InvalidLink:
-        pass
-
-  @classmethod
-  def link_preference(cls, link):
-    return (link.version, isinstance(link, EggLink))
+    return Package.from_href(href, opener=self._crawler.opener)
 
   def iter_unordered(self, req):
     urls = list(itertools.chain(*[fetcher.urls(req) for fetcher in self._fetchers]))
-    for link in filter(None, map(self.translate_href, self._crawler.crawl(*urls))):
-      if link.satisfies(req):
-        yield link
+    for package in filter(None, map(self.translate_href, self._crawler.crawl(*urls))):
+      if package.satisfies(req):
+        yield package
 
   def iter(self, req):
-    """Given a req, return a list of links that satisfy the requirement in best match order."""
-    for link in sorted(self.iter_unordered(req), key=self.link_preference, reverse=True):
-      yield link
+    """Given a req, return a list of packages that satisfy the requirement in best match order."""
+    key = lambda package: self.package_precedence(package, self._precedence)
+    for package in sorted(self.iter_unordered(req), key=key, reverse=True):
+      yield package
 
   def obtain(self, req):
     with TRACER.timed('Obtaining %s' % req):
-      links = list(self.iter(req))
-      TRACER.log('Got ordered links:\n\t%s' % '\n\t'.join(map(str, links)), V=2)
-      for link in links:
-        dist = self._translator.translate(link)
+      packages = list(self.iter(req))
+      for package in packages:
+        dist = self._translator.translate(package)
         if dist:
-          TRACER.log('Picked %s -> %s' % (link, dist), V=2)
           return dist
diff --git a/src/python/twitter/common/python/package.py b/src/python/twitter/common/python/package.py
new file mode 100644
index 000000000..3ef12bf72
--- /dev/null
+++ b/src/python/twitter/common/python/package.py
@@ -0,0 +1,239 @@
+import contextlib
+import os
+import tarfile
+import zipfile
+
+from .base import maybe_requirement
+from .common import safe_mkdtemp
+from .http.link import Link
+from .interpreter import PythonInterpreter
+from .platforms import Platform
+
+from pkg_resources import (
+    EGG_NAME,
+    parse_version,
+    safe_name,
+)
+
+
+class Package(Link):
+  """Base class for named Python binary packages (e.g. source, egg, wheel)."""
+
+  # The registry of concrete implementations
+  _REGISTRY = set()
+
+  @classmethod
+  def register(cls, package_type):
+    """Register a concrete implementation of a Package to be recognized by twitter.common.python."""
+    if not issubclass(package_type, cls):
+      raise TypeError('package_type must be a subclass of Package.')
+    cls._REGISTRY.add(package_type)
+
+  @classmethod
+  def from_href(cls, href, **kw):
+    """Convert from a url to Package.
+
+    :param href: The url to parse
+    :type href: string
+    :returns: A Package object if a valid concrete implementation exists, otherwise None.
+    """
+    for package_type in cls._REGISTRY:
+      try:
+        return package_type(href, **kw)
+      except package_type.InvalidLink:
+        continue
+
+  @property
+  def name(self):
+    return NotImplementedError
+
+  @property
+  def raw_version(self):
+    return NotImplementedError
+
+  @property
+  def version(self):
+    return parse_version(self.raw_version)
+
+  def satisfies(self, requirement):
+    """Determine whether this package matches the requirement.
+
+    :param requirement: The requirement to compare this Package against
+    :type requirement: string or :class:`pkg_resources.Requirement`
+    :returns: True if the package matches the requirement, otherwise False
+    """
+    requirement = maybe_requirement(requirement)
+    link_name = safe_name(self.name).lower()
+    if link_name != requirement.key:
+      return False
+    return self.raw_version in requirement
+
+  def compatible(self, identity, platform=Platform.current()):
+    """Is this link compatible with the given :class:`PythonIdentity` identity and platform?
+
+    :param identity: The Python identity (e.g. CPython 2.7.5) against which compatibility
+    should be checked.
+    :type identity: :class:`PythonIdentity`
+    :param platform: The platform against which compatibility should be checked.  If None, do not
+    check platform compatibility.
+    :type platform: string or None
+    """
+    raise NotImplementedError
+
+
+class SourcePackage(Package):
+  """A Package representing an uncompiled/unbuilt source distribution."""
+
+  EXTENSIONS = {
+    '.tar': (tarfile.TarFile.open, tarfile.ReadError),
+    '.tar.gz': (tarfile.TarFile.open, tarfile.ReadError),
+    '.tar.bz2': (tarfile.TarFile.open, tarfile.ReadError),
+    '.tgz': (tarfile.TarFile.open, tarfile.ReadError),
+    '.zip': (zipfile.ZipFile, zipfile.BadZipfile)
+  }
+
+  @classmethod
+  def split_fragment(cls, fragment):
+    """A heuristic used to split a string into version name/fragment:
+
+       >>> split_fragment('pysolr-2.1.0-beta')
+       ('pysolr', '2.1.0-beta')
+       >>> split_fragment('cElementTree-1.0.5-20051216')
+       ('cElementTree', '1.0.5-20051216')
+       >>> split_fragment('pil-1.1.7b1-20090412')
+       ('pil', '1.1.7b1-20090412')
+       >>> split_fragment('django-plugin-2-2.3')
+       ('django-plugin-2', '2.3')
+    """
+    def likely_version_component(enumerated_fragment):
+      return sum(bool(v and v[0].isdigit()) for v in enumerated_fragment[1].split('.'))
+    fragments = fragment.split('-')
+    if len(fragments) == 1:
+      return fragment, ''
+    max_index, _ = max(enumerate(fragments), key=likely_version_component)
+    return '-'.join(fragments[0:max_index]), '-'.join(fragments[max_index:])
+
+  def __init__(self, url, **kw):
+    super(SourcePackage, self).__init__(url, **kw)
+
+    for ext, class_info in self.EXTENSIONS.items():
+      if self.filename.endswith(ext):
+        self._archive_class = class_info
+        fragment = self.filename[:-len(ext)]
+        break
+    else:
+      raise self.InvalidLink('%s does not end with any of: %s' % (
+          self.filename, ' '.join(self.EXTENSIONS)))
+    self._name, self._raw_version = self.split_fragment(fragment)
+
+  @property
+  def name(self):
+    return safe_name(self._name)
+
+  @property
+  def raw_version(self):
+    return safe_name(self._raw_version)
+
+  @classmethod
+  def first_nontrivial_dir(cls, path):
+    files = os.listdir(path)
+    if len(files) == 1 and os.path.isdir(os.path.join(path, files[0])):
+      return cls.first_nontrivial_dir(os.path.join(path, files[0]))
+    else:
+      return path
+
+  def _unpack(self, filename, location=None):
+    path = location or safe_mkdtemp()
+    archive_class, error_class = self._archive_class
+    try:
+      with contextlib.closing(archive_class(filename)) as package:
+        package.extractall(path=path)
+    except error_class:
+      raise self.UnreadableLink('Could not read %s' % self.url)
+    return self.first_nontrivial_dir(path)
+
+  def fetch(self, location=None, conn_timeout=None):
+    """Fetch and unpack this source target into the location.
+
+    :param location: The location into which the archive should be unpacked.  If None, a temporary
+    ephemeral directory will be created.
+    :type location: string or None
+    :param conn_timeout: A connection timeout for the fetch.  If None, a default is used.
+    :type conn_timeout: float or None
+    :returns: The assumed root directory of the package.
+    """
+    target = super(SourcePackage, self).fetch(conn_timeout=conn_timeout)
+    return self._unpack(target, location)
+
+  # SourcePackages are always compatible as they can be translated to a distribution.
+  def compatible(self, identity, platform=Platform.current()):
+    return True
+
+
+class EggPackage(Package):
+  """A Package representing a built egg."""
+
+  def __init__(self, url, **kw):
+    super(EggPackage, self).__init__(url, **kw)
+    filename, ext = os.path.splitext(self.filename)
+    if ext.lower() != '.egg':
+      raise self.InvalidLink('Not an egg: %s' % filename)
+    matcher = EGG_NAME(filename)
+    if not matcher:
+      raise self.InvalidLink('Could not match egg: %s' % filename)
+
+    self._name, self._raw_version, self._py_version, self._platform = matcher.group(
+        'name', 'ver', 'pyver', 'plat')
+
+    if self._raw_version is None or self._py_version is None:
+      raise self.InvalidLink('url with .egg extension but bad name: %s' % url)
+
+  def __hash__(self):
+    return hash((self.name, self.version, self.py_version, self.platform))
+
+  @property
+  def name(self):
+    return safe_name(self._name)
+
+  @property
+  def raw_version(self):
+    return safe_name(self._raw_version)
+
+  @property
+  def py_version(self):
+    return self._py_version
+
+  @property
+  def platform(self):
+    return self._platform
+
+  def compatible(self, identity, platform=Platform.current()):
+    if not Platform.version_compatible(self.py_version, identity.python):
+      return False
+    if not Platform.compatible(self.platform, platform):
+      return False
+    return True
+
+
+Package.register(SourcePackage)
+Package.register(EggPackage)
+
+
+def distribution_compatible(dist, interpreter=None, platform=None):
+  """Is this distribution compatible with the given interpreter/platform combination?
+
+  :param interpreter: The Python interpreter against which compatibility should be checked.  If None
+  specified, the current interpreter is used.
+  :type identity: :class:`PythonInterpreter` or None
+  :param platform: The platform against which compatibility should be checked.  If None, the current
+  platform will be used
+  :type platform: string or None
+  :returns: True if the distribution is compatible, False if it is unrecognized or incompatible.
+  """
+  interpreter = interpreter or PythonInterpreter.get()
+  platform = platform or Platform.current()
+
+  package = Package.from_href(dist.location)
+  if not package:
+    return False
+  return package.compatible(interpreter.identity, platform=platform)
diff --git a/src/python/twitter/common/python/pex_bootstrapper.py b/src/python/twitter/common/python/pex_bootstrapper.py
index 10d6497c0..149434c07 100644
--- a/src/python/twitter/common/python/pex_bootstrapper.py
+++ b/src/python/twitter/common/python/pex_bootstrapper.py
@@ -2,6 +2,8 @@
 import os
 import zipfile
 
+from .finders import register_finders
+
 __all__ = ('bootstrap_pex',)
 
 
diff --git a/src/python/twitter/common/python/pex_builder.py b/src/python/twitter/common/python/pex_builder.py
index 4662ad1a5..b22dff6bb 100644
--- a/src/python/twitter/common/python/pex_builder.py
+++ b/src/python/twitter/common/python/pex_builder.py
@@ -25,7 +25,7 @@
 from .interpreter import PythonInterpreter
 from .marshaller import CodeMarshaller
 from .pex_info import PexInfo
-from .translator import dist_from_egg
+from .tracer import TRACER
 from .util import CacheHelper, DistributionHelper
 
 from pkg_resources import (
@@ -125,38 +125,37 @@ def add_egg(self, egg):
     self.add_distribution(dist)
     self.add_requirement(dist.as_requirement(), dynamic=False, repo=None)
 
-  def _add_dir_egg(self, egg):
-    for root, _, files in os.walk(egg):
+  def _add_dist_dir(self, path, dist_name):
+    for root, _, files in os.walk(path):
       for f in files:
         filename = os.path.join(root, f)
-        relpath = os.path.relpath(filename, egg)
-        target = os.path.join(self._pex_info.internal_cache, os.path.basename(egg), relpath)
+        relpath = os.path.relpath(filename, path)
+        target = os.path.join(self._pex_info.internal_cache, dist_name, relpath)
         self._chroot.link(filename, target)
-    return CacheHelper.dir_hash(egg)
+    return CacheHelper.dir_hash(path)
 
-  def _add_zipped_egg(self, egg):
-    with open_zip(egg) as zf:
+  def _add_dist_zip(self, path, dist_name):
+    with open_zip(path) as zf:
       for name in zf.namelist():
         if name.endswith('/'):
           continue
-        target = os.path.join(self._pex_info.internal_cache, os.path.basename(egg), name)
+        target = os.path.join(self._pex_info.internal_cache, dist_name, name)
         self._chroot.write(zf.read(name), target)
       return CacheHelper.zip_hash(zf)
 
   def _prepare_code_hash(self):
     self._pex_info.code_hash = CacheHelper.pex_hash(self._chroot.path())
 
-  def add_distribution(self, dist):
-    if not dist.location.endswith('.egg'):
-      raise PEXBuilder.InvalidDependency('Non-egg dependencies not yet supported.')
+  def add_distribution(self, dist, dist_name=None):
+    dist_name = dist_name or os.path.basename(dist.location)
 
     if os.path.isdir(dist.location):
-      dist_hash = self._add_dir_egg(dist.location)
+      dist_hash = self._add_dist_dir(dist.location, dist_name)
     else:
-      dist_hash = self._add_zipped_egg(dist.location)
+      dist_hash = self._add_dist_zip(dist.location, dist_name)
 
     # add dependency key so that it can rapidly be retrieved from cache
-    self._pex_info.add_distribution(os.path.basename(dist.location), dist_hash)
+    self._pex_info.add_distribution(dist_name, dist_hash)
 
   def set_executable(self, filename, env_filename=None):
     if env_filename is None:
@@ -189,28 +188,36 @@ def _prepare_manifest(self):
   def _prepare_main(self):
     self._chroot.write(BOOTSTRAP_ENVIRONMENT, '__main__.py', label='main')
 
-  # TODO(wickman) We should be including _markerlib from setuptools
   # TODO(wickman) Ideally we unqualify our setuptools dependency and inherit whatever is
   # bundled into the environment so long as it is compatible (and error out if not.)
+  #
+  # As it stands, we're picking and choosing the pieces we think we need, which means
+  # if there are bits of setuptools imported from elsewhere they may be incompatible with
+  # this.
   def _prepare_bootstrap(self):
     """
       Write enough of distribute into the .pex .bootstrap directory so that
       we can be fully self-contained.
     """
-    setuptools = dist_from_egg(self._interpreter.get_location('setuptools'))
+    wrote_setuptools = False
+    setuptools = DistributionHelper.distribution_from_path(
+        self._interpreter.get_location('setuptools'))
+
     for fn, content_stream in DistributionHelper.walk_data(setuptools):
-      if fn == 'pkg_resources.py':
-        self._chroot.write(content_stream.read(),
-            os.path.join(self.BOOTSTRAP_DIR, 'pkg_resources.py'), 'resource')
-        break
-    else:
+      if fn == 'pkg_resources.py' or fn.startswith('_markerlib'):
+        self._chroot.write(content_stream.read(), os.path.join(self.BOOTSTRAP_DIR, fn), 'resource')
+        wrote_setuptools = True
+
+    if not wrote_setuptools:
       raise RuntimeError(
           'Failed to extract pkg_resources from setuptools.  Perhaps pants was linked with an '
           'incompatible setuptools.')
+
     libraries = (
       'twitter.common.python',
       'twitter.common.python.http',
     )
+
     for name in libraries:
       dirname = name.replace('twitter.common.python', '_twitter_common_python').replace('.', '/')
       provider = get_provider(name)
diff --git a/src/python/twitter/common/python/platforms.py b/src/python/twitter/common/python/platforms.py
index db78cb90d..41bfea89c 100644
--- a/src/python/twitter/common/python/platforms.py
+++ b/src/python/twitter/common/python/platforms.py
@@ -75,11 +75,3 @@ def compatible(package, platform):
   @staticmethod
   def version_compatible(package_py_version, py_version):
     return package_py_version is None or py_version is None or package_py_version == py_version
-
-  @staticmethod
-  def distribution_compatible(dist, python=None, platform=None):
-    python = python or Platform.python()
-    platform = platform or Platform.current()
-    assert hasattr(dist, 'py_version') and hasattr(dist, 'platform')
-    return Platform.version_compatible(dist.py_version, python) and (
-      Platform.compatible(dist.platform, platform))
diff --git a/src/python/twitter/common/python/resolver.py b/src/python/twitter/common/python/resolver.py
index a8a4b42ce..d47d5e907 100644
--- a/src/python/twitter/common/python/resolver.py
+++ b/src/python/twitter/common/python/resolver.py
@@ -5,11 +5,12 @@
 from .http import Crawler
 from .interpreter import PythonInterpreter
 from .obtainer import Obtainer
+from .package import distribution_compatible
 from .platforms import Platform
 from .translator import (
     ChainedTranslator,
     EggTranslator,
-    SourceTranslator,
+    Translator,
 )
 
 from pkg_resources import (
@@ -19,8 +20,13 @@
 
 
 class ResolverEnvironment(Environment):
+  def __init__(self, interpreter, *args, **kw):
+    kw['python'] = interpreter.python
+    self.__interpreter = interpreter
+    super(ResolverEnvironment, self).__init__(*args, **kw)
+
   def can_add(self, dist):
-    return Platform.distribution_compatible(dist, python=self.python, platform=self.platform)
+    return distribution_compatible(dist, self.__interpreter, platform=self.platform)
 
 
 def requirement_is_exact(req):
@@ -60,12 +66,16 @@ def resolve(requirements,
   platform = platform or Platform.current()
 
   # wire up translators / obtainer
-  shared_options = dict(install_cache=cache, platform=platform)
-  egg_translator = EggTranslator(python=interpreter.python, **shared_options)
-  cache_obtainer = Obtainer(crawler, [Fetcher([cache])], egg_translator) if cache else None
-  source_translator = SourceTranslator(interpreter=interpreter, **shared_options)
-  translator = ChainedTranslator(egg_translator, source_translator)
-  obtainer = Obtainer(crawler, fetchers, translator)
+  if cache:
+    shared_options = dict(install_cache=cache, platform=platform, interpreter=interpreter)
+    translator = EggTranslator(**shared_options)
+    cache_obtainer = Obtainer(crawler, [Fetcher([cache])], translator)
+  else:
+    cache_obtainer = None
+
+  if not obtainer:
+    translator = Translator.default(install_cache=cache, platform=platform, interpreter=interpreter)
+    obtainer = Obtainer(crawler, fetchers, translator)
 
   # make installer
   def installer(req):
@@ -77,5 +87,5 @@ def installer(req):
 
   # resolve
   working_set = WorkingSet(entries=[])
-  env = ResolverEnvironment(search_path=[], platform=platform, python=interpreter.python)
+  env = ResolverEnvironment(interpreter, search_path=[], platform=platform)
   return working_set.resolve(requirements, env=env, installer=installer)
diff --git a/src/python/twitter/common/python/translator.py b/src/python/twitter/common/python/translator.py
index 3bff99ec8..566293fe2 100644
--- a/src/python/twitter/common/python/translator.py
+++ b/src/python/twitter/common/python/translator.py
@@ -2,17 +2,20 @@
 
 from abc import abstractmethod
 import os
-from zipimport import zipimporter
+import warnings
 
 from .common import chmod_plus_w, safe_rmtree, safe_mkdir, safe_mkdtemp
 from .compatibility import AbstractClass
-from .http import EggLink, SourceLink
-from .installer import Installer, EggInstaller
+from .installer import EggInstaller
 from .interpreter import PythonInterpreter
+from .package import (
+    EggPackage,
+    Package,
+    SourcePackage,
+)
 from .platforms import Platform
 from .tracer import TRACER
-
-from pkg_resources import Distribution, EggMetadata, PathMetadata
+from .util import DistributionHelper
 
 
 class TranslatorBase(AbstractClass):
@@ -35,22 +38,13 @@ def __init__(self, *translators):
       if not isinstance(tx, TranslatorBase):
         raise ValueError('Expected a sequence of translators, got %s instead.' % type(tx))
 
-  def translate(self, link):
+  def translate(self, package):
     for tx in self._translators:
-      dist = tx.translate(link)
+      dist = tx.translate(package)
       if dist:
         return dist
 
 
-def dist_from_egg(egg_path):
-  if os.path.isdir(egg_path):
-    metadata = PathMetadata(egg_path, os.path.join(egg_path, 'EGG-INFO'))
-  else:
-    # Assume it's a file or an internal egg
-    metadata = EggMetadata(zipimporter(egg_path))
-  return Distribution.from_filename(egg_path, metadata=metadata)
-
-
 class SourceTranslator(TranslatorBase):
   @classmethod
   def run_2to3(cls, path):
@@ -73,49 +67,51 @@ def __init__(self,
                interpreter=PythonInterpreter.get(),
                platform=Platform.current(),
                use_2to3=False,
-               conn_timeout=None):
+               conn_timeout=None,
+               installer_impl=EggInstaller):
     self._interpreter = interpreter
+    self._installer_impl = installer_impl
     self._use_2to3 = use_2to3
     self._install_cache = install_cache or safe_mkdtemp()
     safe_mkdir(self._install_cache)
     self._conn_timeout = conn_timeout
     self._platform = platform
 
-  def translate(self, link):
-    """From a link, translate a distribution."""
-    if not isinstance(link, SourceLink):
+  def translate(self, package):
+    """From a SourcePackage, translate to a binary distribution."""
+    if not isinstance(package, SourcePackage):
       return None
 
     unpack_path, installer = None, None
     version = self._interpreter.version
 
     try:
-      unpack_path = link.fetch(conn_timeout=self._conn_timeout)
-    except link.UnreadableLink as e:
-      TRACER.log('Failed to fetch %s: %s' % (link, e))
+      unpack_path = package.fetch(conn_timeout=self._conn_timeout)
+    except package.UnreadableLink as e:
+      TRACER.log('Failed to fetch %s: %s' % (package, e))
       return None
 
     try:
       if self._use_2to3 and version >= (3,):
-        with TRACER.timed('Translating 2->3 %s' % link.name):
+        with TRACER.timed('Translating 2->3 %s' % package.name):
           self.run_2to3(unpack_path)
-      # TODO(wickman) Allow for pluggable installers (e.g. WheelInstaller) once
-      # Platform.distribution_compatible understands PEP425 tags.
-      installer = EggInstaller(
+      installer = self._installer_impl(
           unpack_path,
           interpreter=self._interpreter,
-          strict=(link.name != 'distribute'))
-      with TRACER.timed('Packaging %s' % link.name):
+          strict=(package.name not in ('distribute', 'setuptools')))
+      with TRACER.timed('Packaging %s' % package.name):
         try:
           dist_path = installer.bdist()
-        except Installer.InstallFailure:
+        except self._installer_impl.InstallFailure:
           return None
         target_path = os.path.join(self._install_cache, os.path.basename(dist_path))
         os.rename(dist_path, target_path)
-        dist = dist_from_egg(target_path)
-        if Platform.distribution_compatible(
-            dist, python=self._interpreter.python, platform=self._platform):
-          return dist
+        target_package = Package.from_href(target_path)
+        if not target_package:
+          return None
+        if not target_package.compatible(self._interpreter.identity, platform=self._platform):
+          return None
+        return DistributionHelper.distribution_from_path(target_path)
     finally:
       if installer:
         installer.cleanup()
@@ -123,29 +119,41 @@ def translate(self, link):
         safe_rmtree(unpack_path)
 
 
-class EggTranslator(TranslatorBase):
+class BinaryTranslator(TranslatorBase):
   def __init__(self,
+               package_type,
                install_cache=None,
+               interpreter=PythonInterpreter.get(),
                platform=Platform.current(),
-               python=Platform.python(),
+               python=None,
                conn_timeout=None):
+    if python:
+      warnings.warn('python= keyword argument to Translator is deprecated.')
+      if python != interpreter.python:
+        raise ValueError('Two different python interpreters supplied!')
+    self._package_type = package_type
     self._install_cache = install_cache or safe_mkdtemp()
     self._platform = platform
-    self._python = python
+    self._identity = interpreter.identity
     self._conn_timeout = conn_timeout
 
-  def translate(self, link):
-    """From a link, translate a distribution."""
-    if not isinstance(link, EggLink):
+  def translate(self, package):
+    """From a binary package, translate to a local binary distribution."""
+    if not isinstance(package, self._package_type):
       return None
-    if not Platform.distribution_compatible(link, python=self._python, platform=self._platform):
+    if not package.compatible(identity=self._identity, platform=self._platform):
       return None
     try:
-      egg = link.fetch(location=self._install_cache, conn_timeout=self._conn_timeout)
-    except link.UnreadableLink as e:
-      TRACER.log('Failed to fetch %s: %s' % (link, e))
+      bdist = package.fetch(location=self._install_cache, conn_timeout=self._conn_timeout)
+    except package.UnreadableLink as e:
+      TRACER.log('Failed to fetch %s: %s' % (package, e))
       return None
-    return dist_from_egg(egg)
+    return DistributionHelper.distribution_from_path(bdist)
+
+
+class EggTranslator(BinaryTranslator):
+  def __init__(self, **kw):
+    super(EggTranslator, self).__init__(EggPackage, **kw)
 
 
 class Translator(object):
@@ -155,15 +163,11 @@ def default(install_cache=None,
               interpreter=PythonInterpreter.get(),
               conn_timeout=None):
 
-    egg_translator = EggTranslator(
-        install_cache=install_cache,
-        platform=platform,
-        python=interpreter.python,
-        conn_timeout=conn_timeout)
-
-    source_translator = SourceTranslator(
+    shared_options = dict(
         install_cache=install_cache,
         interpreter=interpreter,
         conn_timeout=conn_timeout)
 
+    egg_translator = EggTranslator(platform=platform, **shared_options)
+    source_translator = SourceTranslator(**shared_options)
     return ChainedTranslator(egg_translator, source_translator)
diff --git a/src/python/twitter/common/python/util.py b/src/python/twitter/common/python/util.py
index a01372a88..a3e6adcc2 100644
--- a/src/python/twitter/common/python/util.py
+++ b/src/python/twitter/common/python/util.py
@@ -7,14 +7,10 @@
 import shutil
 import uuid
 
-from pkg_resources import (
-    Distribution,
-    EggMetadata,
-    PathMetadata,
-    find_distributions,
-)
+from pkg_resources import find_distributions
 
 from .common import safe_open, safe_rmtree
+from .finders import register_finders
 
 
 class DistributionHelper(object):
@@ -31,28 +27,28 @@ def walk_data(dist, path='/'):
 
   @staticmethod
   def zipsafe(dist):
-    """Returns whether or not we determine a distribution is zip-safe.
-
-       Only works for egg distributions."""
-    egg_metadata = dist.metadata_listdir('')
-    return 'zip-safe' in egg_metadata and 'native_libs.txt' not in egg_metadata
+    """Returns whether or not we determine a distribution is zip-safe."""
+    # zip-safety is only an attribute of eggs.  wheels are considered never
+    # zip safe per implications of PEP 427.
+    if hasattr(dist, 'egg_info') and dist.egg_info.endswith('EGG-INFO'):
+      egg_metadata = dist.metadata_listdir('')
+      return 'zip-safe' in egg_metadata and 'native_libs.txt' not in egg_metadata
+    else:
+      return False
 
   @classmethod
-  def distribution_from_path(cls, location, location_base=None):
+  def distribution_from_path(cls, path):
     """Returns a Distribution given a location.
 
-       If the distribution name should be based off a different egg name than
-       described by the location, supply location_base as an alternate name, e.g.
-       DistributionHelper.distribution_from_path('/path/to/wrong_package_name-3.2.1',
-           'right_package_name-1.2.3.egg')
+       If no distributions found or if the path contains multiple ambiguous
+       distributions, returns None.
     """
-    location_base = location_base or os.path.basename(location)
-    if os.path.isdir(location):
-      metadata = PathMetadata(location, os.path.join(location, 'EGG-INFO'))
-    else:
-      from zipimport import zipimporter
-      metadata = EggMetadata(zipimporter(location))
-    return Distribution.from_location(location, location_base, metadata=metadata)
+    # Monkeypatch pkg_resources finders should it not already be so.
+    register_finders()
+    distributions = list(find_distributions(path))
+    if len(distributions) != 1:
+      return None
+    return distributions[0]
 
 
 class CacheHelper(object):
@@ -137,6 +133,7 @@ def cache_distribution(cls, zf, source, target_dir):
           safe_rmtree(target_dir_tmp)
         else:
           raise
-    distributions = list(find_distributions(target_dir))
-    assert len(distributions) == 1, 'Failed to cache distribution %s' % source
-    return distributions[0]
+
+    dist = DistributionHelper.distribution_from_path(target_dir)
+    assert dist is not None, 'Failed to cache distribution %s' % source
+    return dist
diff --git a/tests/python/twitter/common/python/BUILD b/tests/python/twitter/common/python/BUILD
index f069102c8..1eeab75a1 100644
--- a/tests/python/twitter/common/python/BUILD
+++ b/tests/python/twitter/common/python/BUILD
@@ -14,9 +14,6 @@
 # limitations under the License.
 # ==================================================================================================
 
-def mock_filter(python, platform):
-  return python.startswith('2')
-
 
 python_test_suite(name = 'all',
   dependencies = [
@@ -25,6 +22,7 @@ python_test_suite(name = 'all',
     pants(':test_finders'),
     pants(':test_interpreter'),
     pants(':test_obtainer'),
+    pants(':test_package'),
     pants(':test_pep425'),
     pants(':test_platform'),
     pants(':test_pex_builder'),
@@ -44,7 +42,7 @@ python_tests(name = 'test_finders',
   sources = ['test_finders.py'],
   dependencies = [
     pants('src/python/twitter/common/python'),
-    python_requirement('mock', version_filter=mock_filter)
+    python_requirement('mock', version_filter=lambda py, plat: py.startswith('2'))
   ]
 )
 
@@ -63,10 +61,11 @@ python_tests(name = 'test_obtainer',
   ]
 )
 
-python_tests(name = 'test_platform',
-  sources = ['test_platform.py'],
+python_tests(name = 'test_package',
+  sources = ['test_package.py'],
   dependencies = [
-    pants('src/python/twitter/common/python')
+    pants('src/python/twitter/common/contextutil'),
+    pants('src/python/twitter/common/python'),
   ]
 )
 
@@ -77,6 +76,13 @@ python_tests(name = 'test_pep425',
   ]
 )
 
+python_tests(name = 'test_platform',
+  sources = ['test_platform.py'],
+  dependencies = [
+    pants('src/python/twitter/common/python')
+  ]
+)
+
 python_tests(name = 'test_pex_builder',
   sources = ['test_pex_builder.py'],
   dependencies = [
diff --git a/tests/python/twitter/common/python/http/BUILD b/tests/python/twitter/common/python/http/BUILD
index ac913c7a9..abbdd6506 100644
--- a/tests/python/twitter/common/python/http/BUILD
+++ b/tests/python/twitter/common/python/http/BUILD
@@ -18,11 +18,11 @@ python_test_suite(name = 'all',
   dependencies = [
     pants(':test_crawler'),
     pants(':test_http'),
-    pants(':test_link'),
   ]
 )
 
-python_tests(name = 'test_crawler',
+python_tests(
+  name = 'test_crawler',
   sources = ['test_crawler.py'],
   dependencies = [
     pants('src/python/twitter/common/contextutil'),
@@ -30,12 +30,11 @@ python_tests(name = 'test_crawler',
   ]
 )
 
-
 def mock_filter(python, platform):
   return python.startswith('2')
 
-
-python_tests(name = 'test_http',
+python_tests(
+  name = 'test_http',
   sources = ['test_http.py'],
   dependencies = [
     pants('src/python/twitter/common/contextutil'),
@@ -45,11 +44,3 @@ python_tests(name = 'test_http',
     python_requirement('mock', version_filter=mock_filter)
   ]
 )
-
-python_tests(name = 'test_link',
-  sources = ['test_link.py'],
-  dependencies = [
-    pants('src/python/twitter/common/contextutil'),
-    pants('src/python/twitter/common/python'),
-  ]
-)
diff --git a/tests/python/twitter/common/python/test_obtainer.py b/tests/python/twitter/common/python/test_obtainer.py
index 0359ae963..3caa01a06 100644
--- a/tests/python/twitter/common/python/test_obtainer.py
+++ b/tests/python/twitter/common/python/test_obtainer.py
@@ -15,23 +15,26 @@
 # ==================================================================================================
 
 from twitter.common.python.fetcher import Fetcher
-from twitter.common.python.http import EggLink, SourceLink
+from twitter.common.python.package import EggPackage, SourcePackage
 from twitter.common.python.obtainer import Obtainer
+
 from pkg_resources import Requirement
 
 
-def test_link_preference():
-  sl = SourceLink('psutil-0.6.1.tar.gz')
-  el = EggLink('psutil-0.6.1-py2.6.egg')
-  assert Obtainer.link_preference(el) > Obtainer.link_preference(sl)
+def test_package_precedence():
+  source = SourcePackage('psutil-0.6.1.tar.gz')
+  egg = EggPackage('psutil-0.6.1-py2.6.egg')
 
+  # default precedence
+  assert Obtainer.package_precedence(egg) > Obtainer.package_precedence(source)
 
-class FakeObtainer(Obtainer):
-  def __init__(self, links):
-    self._links = list(links)
+  # overridden precedence
+  PRECEDENCE = (EggPackage,)
+  assert Obtainer.package_precedence(source, PRECEDENCE) == (source.version, -1)  # unknown rank
 
-  def iter_unordered(self, req):
-    return self._links
+  PRECEDENCE = (SourcePackage, EggPackage)
+  assert Obtainer.package_precedence(source, PRECEDENCE) > Obtainer.package_precedence(
+      egg, PRECEDENCE)
 
 
 class FakeCrawler(object):
@@ -43,29 +46,41 @@ def crawl(self, *args, **kw):
     return self._hrefs
 
 
+class FakeObtainer(Obtainer):
+  def __init__(self, links):
+    self.__links = list(links)
+    super(FakeObtainer, self).__init__(FakeCrawler([]), [], [])
+
+  def iter_unordered(self, req):
+    return iter(self.__links)
+
+
 def test_iter_ordering():
-  PS, PS_EGG = SourceLink('psutil-0.6.1.tar.gz'), EggLink('psutil-0.6.1-py3.3-linux-x86_64.egg')
-  PS_REQ = Requirement.parse('psutil')
+  tgz = SourcePackage('psutil-0.6.1.tar.gz')
+  egg = EggPackage('psutil-0.6.1-py3.3-linux-x86_64.egg')
+  req = Requirement.parse('psutil')
 
-  assert list(FakeObtainer([PS, PS_EGG]).iter(PS_REQ)) == [PS_EGG, PS]
-  assert list(FakeObtainer([PS_EGG, PS]).iter(PS_REQ)) == [PS_EGG, PS]
+  assert list(FakeObtainer([tgz, egg]).iter(req)) == [egg, tgz]
+  assert list(FakeObtainer([egg, tgz]).iter(req)) == [egg, tgz]
 
 
 def test_href_translation():
   VERSIONS = ['0.4.0', '0.4.1', '0.5.0', '0.6.0']
+
   def fake_link(version):
     return 'http://www.example.com/foo/bar/psutil-%s.tar.gz' % version
+
   fc = FakeCrawler([fake_link(v) for v in VERSIONS])
   ob = Obtainer(fc, [], [])
 
   for v in VERSIONS:
     pkgs = list(ob.iter(Requirement.parse('psutil==%s' % v)))
     assert len(pkgs) == 1, 'Version: %s' % v
-    assert pkgs[0] == SourceLink(fake_link(v))
+    assert pkgs[0] == SourcePackage(fake_link(v))
 
   assert list(ob.iter(Requirement.parse('psutil>=0.5.0'))) == [
-    SourceLink(fake_link('0.6.0')),
-    SourceLink(fake_link('0.5.0'))]
+    SourcePackage(fake_link('0.6.0')),
+    SourcePackage(fake_link('0.5.0'))]
 
   assert list(ob.iter(Requirement.parse('psutil'))) == [
-      SourceLink(fake_link(v)) for v in reversed(VERSIONS)]
+      SourcePackage(fake_link(v)) for v in reversed(VERSIONS)]
diff --git a/tests/python/twitter/common/python/http/test_link.py b/tests/python/twitter/common/python/test_package.py
similarity index 74%
rename from tests/python/twitter/common/python/http/test_link.py
rename to tests/python/twitter/common/python/test_package.py
index 517f1a8c0..36028207d 100644
--- a/tests/python/twitter/common/python/http/test_link.py
+++ b/tests/python/twitter/common/python/test_package.py
@@ -1,23 +1,20 @@
 import contextlib
 import os
-import pytest
+
 from zipfile import ZipFile
 
 from twitter.common.contextutil import temporary_dir
-from twitter.common.python.http import (
-  EggLink,
-  Link,
-  SourceLink,
-  Web,
-)
+from twitter.common.python.http import Web
+from twitter.common.python.package import EggPackage, SourcePackage
 from twitter.common.python.testing import create_layout
 
 from pkg_resources import Requirement, parse_version
+import pytest
 
 
-def test_source_links():
+def test_source_packages():
   for ext in ('.tar.gz', '.tar', '.tgz', '.zip', '.tar.bz2'):
-    sl = SourceLink('a_p_r-3.1.3' + ext)
+    sl = SourcePackage('a_p_r-3.1.3' + ext)
     assert sl._name == 'a_p_r'
     assert sl.name == 'a-p-r'
     assert sl.raw_version == '3.1.3'
@@ -27,7 +24,7 @@ def test_source_links():
       assert sl.satisfies(Requirement.parse(req))
     for req in ('foo', 'a_p_r==4.0.0', 'a_p_r>4.0.0', 'a_p_r>3.0.0,<3.0.3', 'a==3.1.3'):
       assert not sl.satisfies(req)
-  sl = SourceLink('python-dateutil-1.5.tar.gz')
+  sl = SourcePackage('python-dateutil-1.5.tar.gz')
   assert sl.name == 'python-dateutil'
   assert sl.raw_version == '1.5'
 
@@ -37,15 +34,15 @@ def test_source_links():
     with contextlib.closing(ZipFile(os.path.join(td, dateutil), 'w')) as zf:
       zf.writestr(os.path.join(dateutil_base, 'file1.txt'), 'junk1')
       zf.writestr(os.path.join(dateutil_base, 'file2.txt'), 'junk2')
-    sl = SourceLink('file://' + os.path.join(td, dateutil), opener=Web())
+    sl = SourcePackage('file://' + os.path.join(td, dateutil), opener=Web())
     with temporary_dir() as td2:
       sl.fetch(location=td2)
       print(os.listdir(td2))
       assert set(os.listdir(os.path.join(td2, dateutil_base))) == set(['file1.txt', 'file2.txt'])
 
 
-def test_egg_links():
-  el = EggLink('psutil-0.4.1-py2.6-macosx-10.7-intel.egg')
+def test_egg_packages():
+  el = EggPackage('psutil-0.4.1-py2.6-macosx-10.7-intel.egg')
   assert el.name == 'psutil'
   assert el.raw_version == '0.4.1'
   assert el.py_version == '2.6'
@@ -55,25 +52,26 @@ def test_egg_links():
   for req in ('foo', 'bar==0.4.1'):
     assert not el.satisfies(req)
 
-  el = EggLink('pytz-2012b-py2.6.egg')
+  el = EggPackage('pytz-2012b-py2.6.egg')
   assert el.name == 'pytz'
   assert el.raw_version == '2012b'
   assert el.py_version == '2.6'
   assert el.platform is None
 
   # Eggs must have their own version and a python version.
-  with pytest.raises(Link.InvalidLink):
-    EggLink('bar.egg')
+  with pytest.raises(EggPackage.InvalidLink):
+    EggPackage('bar.egg')
 
-  with pytest.raises(Link.InvalidLink):
-    EggLink('bar-1.egg')
+  with pytest.raises(EggPackage.InvalidLink):
+    EggPackage('bar-1.egg')
 
-  with pytest.raises(Link.InvalidLink):
-    EggLink('bar-py2.6.egg')
+  with pytest.raises(EggPackage.InvalidLink):
+    EggPackage('bar-py2.6.egg')
 
   dateutil = 'python_dateutil-1.5-py2.6.egg'
   with create_layout([dateutil]) as td:
-    el = EggLink('file://' + os.path.join(td, dateutil), opener=Web())
+    el = EggPackage('file://' + os.path.join(td, dateutil), opener=Web())
+
     with temporary_dir() as td2:
       # local file fetch w/o location will always remain same
       loc1 = el.fetch()
diff --git a/tests/python/twitter/common/python/test_util.py b/tests/python/twitter/common/python/test_util.py
index 13418867d..0290dcc3e 100644
--- a/tests/python/twitter/common/python/test_util.py
+++ b/tests/python/twitter/common/python/test_util.py
@@ -1,4 +1,5 @@
 import contextlib
+import functools
 from hashlib import sha1
 import os
 import random
@@ -7,7 +8,7 @@
 
 from twitter.common.contextutil import temporary_file, temporary_dir
 from twitter.common.dirutil import safe_mkdir, safe_mkdtemp
-from twitter.common.python.installer import Installer
+from twitter.common.python.installer import Installer, EggInstaller
 from twitter.common.python.testing import (
     make_distribution,
     temporary_content,
@@ -60,6 +61,9 @@ def test_hash_consistency():
 
 
 def test_zipsafe():
+  make_egg = functools.partial(make_distribution, installer_impl=EggInstaller)
+
   for zipped in (False, True):
-    with make_distribution(zipped=zipped) as dist:
-      assert DistributionHelper.zipsafe(dist)
+    for zip_safe in (False, True):
+      with make_egg(zipped=zipped, zip_safe=zip_safe) as dist:
+        assert DistributionHelper.zipsafe(dist) is zip_safe