Skip to content

Commit

Permalink
Port pants to pex 0.8.x
Browse files Browse the repository at this point in the history
SSIA.  The user facing parts of this change:
  - package resolution happens via requests now -- this is way more reliable
    and results in fewer Untranslateable exceptions
  - PEX_VERBOSE now controls all pex verbosity, including better messages around Untranslateable (yay!)
  - fixes a major pex regression with namespace packages introduced somewhere in 0.5.x

The pex 0.7.0 -> 0.8.0 changelog:

* *API change*: Decouple translation from package iteration.  This removes
  the Obtainer construct entirely, which likely means if you're using PEX as
  a library, you will need to change your code if you were doing anything
  nontrivial.  This adds a couple new options to ``resolve`` but simplifies
  the story around how to cache packages.
  [RB pantsbuild#785](https://rbcommons.com/s/twitter/r/785/)
* Refactor http handling in pex to allow for alternate http implementations.  Adds support
  for [requests](https://github.com/kennethreitz/requests),
  improving both performance and security.   For more information, read the commit notes at
  [91c7f32](pex-tool/pex@91c7f32).
  [RB pantsbuild#778](https://rbcommons.com/s/twitter/r/778/)
* Improvements to API documentation throughout.
* Renamed ``Tracer`` to ``TraceLogger`` to prevent nondeterministic isort ordering.
* Refactor tox.ini to increase the number of environment combinations and improve coverage.
* Adds HTTP retry support for the RequestsContext.
  [RB pantsbuild#1303](https://rbcommons.com/s/twitter/r/1303/)
* Make pex --version correct.
  [Issue #19](pex-tool/pex#19)
* Bug fix: Fix over-aggressive sys.modules scrubbing for namespace packages.  Under
  certain circumstances, namespace packages in site-packages could conflict with packages
  within a PEX, causing them to fail importing.
  [RB pantsbuild#1378](https://rbcommons.com/s/twitter/r/1378/)
* Bug fix: Replace uses of ``os.unsetenv(...)`` with ``del os.environ[...]``
  [Pull Request #11](pex-tool/pex#11)
* Bug fix: Scrub sys.path and sys.modules based upon both supplied path and
  realpath of files and directories.  Newer versions of virtualenv on Linux symlink site-packages
  which caused those packages to not be removed from sys.path correctly.
  [Issue #21](pex-tool/pex#21)
* Bug fix: The pex -s option was not correctly pulling in transitive dependencies.
  [Issue #22](pex-tool/pex#22)
* Bug fix: Adds ``content`` method to HTTP contexts that does HTML content decoding, fixing
  an encoding issue only experienced when using Python 3.
  [Issue #10](pex-tool/pex#10)

Testing Done:
build-support/bin/ci.sh

Reviewed at https://rbcommons.com/s/twitter/r/1421/
wickman committed Dec 4, 2014

Verified

This commit was signed with the committer’s verified signature.
bukka Jakub Zelenka
1 parent df75d99 commit 80b9759
Showing 14 changed files with 78 additions and 91 deletions.
4 changes: 3 additions & 1 deletion 3rdparty/python/requirements.txt
Original file line number Diff line number Diff line change
@@ -9,7 +9,9 @@ docutils>=0.12,<0.13
Markdown==2.1.1
mock==1.0.1
mox==0.5.3
pex>=0.7.0,<0.8
# Until https://github.com/pantsbuild/pex/issues/28 is addressed, we cannot
# do pex<0.9.0.
pex>=0.8.1,<0.8.999999
psutil==1.1.3
lockfile==0.10.2
Pygments==1.4
2 changes: 2 additions & 0 deletions build-support/pants_venv
Original file line number Diff line number Diff line change
@@ -50,6 +50,8 @@ function activate_pants_venv() {
# Install as an egg so IDEs can grok all deps. Installing as sources may
# run afoul of this bug: http://youtrack.jetbrains.com/issue/PY-6477.
# TODO: Remove the --egg once this is resolved. It won't work in the wheel world anyway.
# Use -f ${REPO_ROOT}/third_party if patching in local dependencies
# pip_extra="--egg -f ${REPO_ROOT}/third_party"
pip_extra="--egg"
else
pip_extra=""
5 changes: 2 additions & 3 deletions src/python/pants/backend/python/binary_builder.py
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception):
pass

def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None):
def __init__(self, target, run_tracker, interpreter=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
@@ -45,8 +45,7 @@ def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None):
targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter,
conn_timeout=conn_timeout)
interpreter=self.interpreter)

def run(self):
print('Building PythonBinary %s:' % self.target)
27 changes: 17 additions & 10 deletions src/python/pants/backend/python/interpreter_cache.py
Original file line number Diff line number Diff line change
@@ -9,14 +9,15 @@
from pkg_resources import Requirement
import shutil


from pex.archiver import Archiver
from pex.crawler import Crawler
from pex.installer import EggInstaller
from pex.interpreter import PythonIdentity, PythonInterpreter
from pex.obtainer import Obtainer
from pex.iterator import Iterator
from pex.package import EggPackage, SourcePackage

from pants.backend.python.python_setup import PythonSetup
from pants.backend.python.resolver import crawler_from_config, fetchers_from_config
from pants.backend.python.resolver import context_from_config, fetchers_from_config
from pants.util.dirutil import safe_mkdir


@@ -39,33 +40,39 @@ def _resolve_interpreter(config, interpreter, requirement, logger=print):
return interpreter

def installer_provider(sdist):
return EggInstaller(sdist, strict=requirement.key != 'setuptools', interpreter=interpreter)
return EggInstaller(
Archiver.unpack(sdist),
strict=requirement.key != 'setuptools',
interpreter=interpreter)

egg = _resolve_and_link(
config,
requirement,
os.path.join(interpreter_dir, requirement.key),
installer_provider,
logger=logger)

if egg:
return interpreter.with_extra(egg.name, egg.raw_version, egg.url)
return interpreter.with_extra(egg.name, egg.raw_version, egg.path)
else:
logger('Failed to resolve requirement %s for %s' % (requirement, interpreter))


def _resolve_and_link(config, requirement, target_link, installer_provider, logger=print):
# Short-circuit if there is a local copy
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
egg = EggPackage(os.path.realpath(target_link))
if egg.satisfies(requirement):
return egg

fetchers = fetchers_from_config(config)
crawler = crawler_from_config(config)
obtainer = Obtainer(crawler, fetchers, [])
obtainer_iterator = obtainer.iter(requirement)
links = [link for link in obtainer_iterator if isinstance(link, SourcePackage)]
context = context_from_config(config)
iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)]

for link in links:
logger(' fetching %s' % link.url)
sdist = link.fetch()
sdist = context.fetch(link)
logger(' installing %s' % sdist)
installer = installer_provider(sdist)
dist_location = installer.bdist()
8 changes: 2 additions & 6 deletions src/python/pants/backend/python/python_builder.py
Original file line number Diff line number Diff line change
@@ -17,9 +17,7 @@ class PythonBuilder(object):
def __init__(self, run_tracker):
self._run_tracker = run_tracker

def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=False,
debug=False):

def build(self, targets, args, interpreter=None, fast_tests=False, debug=False):
test_targets = []
binary_targets = []
interpreter = interpreter or PythonInterpreter.get()
@@ -38,7 +36,6 @@ def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=F
test_targets,
args,
interpreter=interpreter,
conn_timeout=conn_timeout,
fast=fast_tests,
debug=debug).run()
if rv != 0:
@@ -48,8 +45,7 @@ def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=F
rv = PythonBinaryBuilder(
binary_target,
self._run_tracker,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
interpreter=interpreter).run()
if rv != 0:
return rv

19 changes: 10 additions & 9 deletions src/python/pants/backend/python/python_chroot.py
Original file line number Diff line number Diff line change
@@ -60,16 +60,14 @@ def __init__(self,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
interpreter=None):
self._config = Config.from_cache()
self._targets = targets
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
interpreter=self._interpreter)
self._conn_timeout = conn_timeout

# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
@@ -127,10 +125,9 @@ def copy_to_chroot(base, path, add_function):
resource=resources_tgt.address.spec))
raise

def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' % (str(req),
' (dynamic)' if dynamic else '', ' (repo: %s)' % repo if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_requirement(self, req):
self.debug(' Dumping requirement: %s' % req)
self._builder.add_requirement(req)

def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location))
@@ -194,19 +191,23 @@ def dump(self):
reqs_from_libraries.add(req)

reqs_to_build = OrderedSet()
find_links = []

for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
reqs_to_build.add(req)
self._dump_requirement(req._requirement, False, req._repository)
self._dump_requirement(req.requirement)
if req.repository:
find_links.append(req.repository)

distributions = resolve_multi(
self._config,
reqs_to_build,
interpreter=self._interpreter,
platforms=self._platforms,
conn_timeout=self._conn_timeout)
find_links=find_links)

locations = set()
for platform, dist_set in distributions.items():
59 changes: 20 additions & 39 deletions src/python/pants/backend/python/resolver.py
Original file line number Diff line number Diff line change
@@ -6,9 +6,10 @@
print_function, unicode_literals)

from pex.fetcher import Fetcher, PyPIFetcher
from pex.http import Crawler
from pex.crawler import Crawler
from pex.http import Context
from pex.interpreter import PythonInterpreter
from pex.obtainer import CachingObtainer
from pex.iterator import Iterator
from pex.platforms import Platform
from pex.resolver import resolve
from pex.translator import Translator
@@ -29,33 +30,17 @@ def fetchers_from_config(config):
return fetchers


def crawler_from_config(config, conn_timeout=None):
download_cache = PythonSetup(config).scratch_dir('download_cache', default_name='downloads')
return Crawler(cache=download_cache, conn_timeout=conn_timeout)


class PantsObtainer(CachingObtainer):
def iter(self, requirement):
if hasattr(requirement, 'repository') and requirement.repository:
obtainer = CachingObtainer(
install_cache=self.install_cache,
ttl=self.ttl,
crawler=self._crawler,
fetchers=[Fetcher([requirement.repository])],
translators=self._translator)
for package in obtainer.iter(requirement):
yield package
else:
for package in super(PantsObtainer, self).iter(requirement):
yield package
def context_from_config(config):
# TODO(wickman) Add retry, conn_timeout, threads, etc configuration here.
return Context.get()


def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
ttl=3600,
find_links=None):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
@@ -68,35 +53,31 @@ def resolve_multi(config,
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))

install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
fetchers = fetchers_from_config(config)
if find_links:
fetchers.extend(Fetcher([path]) for path in find_links)
context = context_from_config(config)

for platform in platforms:
translator = Translator.default(
install_cache=install_cache,
distributions[platform] = resolve(
requirements=requirements,
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
conn_timeout=conn_timeout)

obtainer = PantsObtainer(
install_cache=install_cache,
crawler=crawler_from_config(config, conn_timeout=conn_timeout),
fetchers=fetchers_from_config(config) or [PyPIFetcher()],
translators=translator)

distributions[platform] = resolve(requirements=requirements,
obtainer=obtainer,
interpreter=interpreter,
platform=platform)
context=context,
cache=cache,
cache_ttl=ttl)

return distributions
1 change: 0 additions & 1 deletion src/python/pants/backend/python/tasks/pytest_run.py
Original file line number Diff line number Diff line change
@@ -49,7 +49,6 @@ def is_python_test(target):
test_builder = PythonTestBuilder(targets=test_targets,
args=args,
interpreter=self.interpreter,
conn_timeout=self.conn_timeout,
fast=self.get_options().fast,
debug=debug)
with self.context.new_workunit(name='run',
Original file line number Diff line number Diff line change
@@ -54,8 +54,7 @@ def create_binary(self, binary):
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
interpreter=interpreter)

pex_path = os.path.join(self._distdir, '%s.pex' % binary.name)
chroot.dump()
3 changes: 1 addition & 2 deletions src/python/pants/backend/python/tasks/python_repl.py
Original file line number Diff line number Diff line change
@@ -48,8 +48,7 @@ def execute(self):
targets=targets,
extra_requirements=extra_requirements,
builder=builder,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
interpreter=interpreter)

chroot.dump()
builder.freeze()
3 changes: 1 addition & 2 deletions src/python/pants/backend/python/tasks/python_run.py
Original file line number Diff line number Diff line change
@@ -40,8 +40,7 @@ def execute(self):
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
interpreter=interpreter)

chroot.dump()
builder.freeze()
11 changes: 0 additions & 11 deletions src/python/pants/backend/python/tasks/python_task.py
Original file line number Diff line number Diff line change
@@ -17,17 +17,8 @@


class PythonTask(Task):
@classmethod
def register_options(cls, register):
super(PythonTask, cls).register_options(register)
register('--timeout', type=int, default=0,
help='Number of seconds to wait for http connections.')

def __init__(self, *args, **kwargs):
super(PythonTask, self).__init__(*args, **kwargs)
self.conn_timeout = (self.get_options().timeout or
self.context.config.getdefault('connection_timeout'))

self._compatibilities = self.get_options().interpreter or [b'']
self._interpreter_cache = None
self._interpreter = None
@@ -96,5 +87,3 @@ def temporary_pex_builder(self, interpreter=None, pex_info=None, parent_dir=None
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info)
yield builder
builder.chroot().delete()


6 changes: 2 additions & 4 deletions src/python/pants/backend/python/test_builder.py
Original file line number Diff line number Diff line change
@@ -65,11 +65,10 @@ class PythonTestBuilder(object):
PythonRequirement('unittest2py3k', version_filter=lambda py, pl: py.startswith('3'))
]

def __init__(self, targets, args, interpreter=None, conn_timeout=None, fast=False, debug=False):
def __init__(self, targets, args, interpreter=None, fast=False, debug=False):
self._targets = targets
self._args = args
self._interpreter = interpreter or PythonInterpreter.get()
self._conn_timeout = conn_timeout

# If fast is true, we run all the tests in a single chroot. This is MUCH faster than
# creating a chroot for each test target. However running each test separately is more
@@ -292,8 +291,7 @@ def _test_runner(self, targets, stdout, stderr):
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self._interpreter,
conn_timeout=self._conn_timeout)
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
18 changes: 17 additions & 1 deletion tests/python/pants_test/backend/python/test_test_builder.py
Original file line number Diff line number Diff line change
@@ -7,11 +7,13 @@

import glob
import os
import sys
from textwrap import dedent
import xml.dom.minidom as DOM

import coverage

from pants.backend.python.interpreter_cache import PythonInterpreterCache
from pants.backend.python.targets.python_library import PythonLibrary
from pants.backend.python.targets.python_tests import PythonTests
from pants.backend.python.test_builder import PythonTestBuilder
@@ -22,7 +24,21 @@

class PythonTestBuilderTestBase(BaseTest):
def run_tests(self, targets, args=None, fast=True, debug=False):
test_builder = PythonTestBuilder(targets, args or [], fast=fast, debug=debug)
cache = PythonInterpreterCache(self.config())
cache.setup()

interpreter = None

for interp in cache.interpreters:
if interp.binary == sys.executable:
interpreter = interp
break
else:
raise RuntimeError('Could not find suitable interpreter to run tests.')

test_builder = PythonTestBuilder(
targets, args or [], fast=fast, debug=debug, interpreter=interpreter)

with pushd(self.build_root):
return test_builder.run()

0 comments on commit 80b9759

Please sign in to comment.