diff --git a/CHANGES.rst b/CHANGES.rst index 6a90f1c6e..239cd25c9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,9 +3,22 @@ CHANGES ======= ---------- -0.8.6 +0.8.7-dev0 ---------- +* Bug fix: Make ``pip install pex`` work better by removing ``extras_requires`` on the + ``console_script`` entry point. Fixes `#48 `_ + +* New feature: Adds an interpreter cache to the ``pex`` tool. If the user does not explicitly + disable the wheel feature and attempts to build a pex with wheels but does not have the wheel + package installed, pex will download it in order to make the feature work. + Implements `#47 `_ in order to + fix `#48 `_ + +----- +0.8.6 +----- + * Bug fix: Honor installed sys.excepthook in pex teardown. `RB #1733 `_ diff --git a/pex/bin/pex.py b/pex/bin/pex.py index e168a8344..5078a4fd9 100644 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -8,15 +8,21 @@ from __future__ import absolute_import, print_function +import functools import os import shutil import sys from optparse import OptionParser -from pex.common import safe_delete, safe_mkdtemp +from pex.archiver import Archiver +from pex.base import maybe_requirement +from pex.common import safe_delete, safe_mkdir, safe_mkdtemp +from pex.crawler import Crawler from pex.fetcher import Fetcher, PyPIFetcher +from pex.http import Context from pex.installer import EggInstaller, Packager, WheelInstaller from pex.interpreter import PythonInterpreter +from pex.iterator import Iterator from pex.package import EggPackage, Package, SourcePackage, WheelPackage from pex.pex import PEX from pex.pex_builder import PEXBuilder @@ -24,9 +30,10 @@ from pex.resolver import resolve as requirement_resolver from pex.tracer import TRACER, TraceLogger from pex.translator import ChainedTranslator, EggTranslator, SourceTranslator, WheelTranslator -from pex.version import __version__ +from pex.version import __setuptools_requirement, __version__, __wheel_requirement CANNOT_DISTILL = 101 +CANNOT_SETUP_INTERPRETER = 102 def die(msg, error_code=1): @@ -129,12 +136,19 @@ def configure_clp(): help='Inherit the contents of sys.path (including site-packages) running the pex. ' '[Default: %default]') + parser.add_option( + '--interpreter-cache-dir', + dest='interpreter_cache_dir', + default=os.path.expanduser('~/.pex/interpreters'), + help='The interpreter cache to use for keeping track of interpreter dependencies ' + 'for the pex tool. [Default: %default]') + parser.add_option( '--cache-dir', dest='cache_dir', default=os.path.expanduser('~/.pex/build'), help='The local cache directory to use for speeding up requirement ' - 'lookups; [Default: %default]') + 'lookups. [Default: %default]') parser.add_option( '--cache-ttl', @@ -202,8 +216,87 @@ def configure_clp(): return parser +def _safe_link(src, dst): + try: + os.unlink(dst) + except OSError: + pass + os.symlink(src, dst) + + +def _resolve_and_link_interpreter(requirement, fetchers, target_link, installer_provider): + # Short-circuit if there is a local copy + if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)): + egg = EggPackage(os.path.realpath(target_link)) + if egg.satisfies(requirement): + return egg + + context = Context.get() + iterator = Iterator(fetchers=fetchers, crawler=Crawler(context)) + links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)] + + with TRACER.timed('Interpreter cache resolving %s' % requirement, V=2): + for link in links: + with TRACER.timed('Fetching %s' % link, V=3): + sdist = context.fetch(link) + + with TRACER.timed('Installing %s' % link, V=3): + installer = installer_provider(sdist) + dist_location = installer.bdist() + target_location = os.path.join( + os.path.dirname(target_link), os.path.basename(dist_location)) + shutil.move(dist_location, target_location) + _safe_link(target_location, target_link) + + return EggPackage(target_location) + + +def resolve_interpreter(cache, fetchers, interpreter, requirement): + """Resolve an interpreter with a specific requirement. + + Given a :class:`PythonInterpreter` and a requirement, return an + interpreter with the capability of resolving that requirement or + ``None`` if it's not possible to install a suitable requirement.""" + requirement = maybe_requirement(requirement) + + # short circuit + if interpreter.satisfies([requirement]): + return interpreter + + def installer_provider(sdist): + return EggInstaller( + Archiver.unpack(sdist), + strict=requirement.key != 'setuptools', + interpreter=interpreter) + + interpreter_dir = os.path.join(cache, str(interpreter.identity)) + safe_mkdir(interpreter_dir) + + egg = _resolve_and_link_interpreter( + requirement, + fetchers, + os.path.join(interpreter_dir, requirement.key), + installer_provider) + + if egg: + return interpreter.with_extra(egg.name, egg.raw_version, egg.path) + + +def fetchers_from_options(options): + fetchers = [Fetcher(options.repos)] + + if options.pypi: + fetchers.append(PyPIFetcher()) + + if options.indices: + fetchers.extend(PyPIFetcher(index) for index in options.indices) + + return fetchers + + def interpreter_from_options(options): interpreter = None + if options.python: if os.path.exists(options.python): interpreter = PythonInterpreter.from_binary(options.python) @@ -213,11 +306,23 @@ def interpreter_from_options(options): die('Failed to find interpreter: %s' % options.python) else: interpreter = PythonInterpreter.get() - return interpreter + with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): + fetchers = fetchers_from_options(options) + + resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, fetchers) + + # resolve setuptools + interpreter = resolve(interpreter, __setuptools_requirement) + + # possibly resolve wheel + if interpreter and options.use_wheel: + interpreter = resolve(interpreter, __wheel_requirement) + + return interpreter -def translator_from_options(options): - interpreter = interpreter_from_options(options) + +def translator_from_options(interpreter, options): platform = options.platform translators = [] @@ -237,7 +342,11 @@ def translator_from_options(options): def build_pex(args, options): - interpreter = interpreter_from_options(options) + with TRACER.timed('Resolving interpreter', V=2): + interpreter = interpreter_from_options(options) + + if interpreter is None: + die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder( path=safe_mkdtemp(), @@ -253,17 +362,8 @@ def build_pex(args, options): installer = WheelInstaller if options.use_wheel else EggInstaller - interpreter = interpreter_from_options(options) - - fetchers = [Fetcher(options.repos)] - - if options.pypi: - fetchers.append(PyPIFetcher()) - - if options.indices: - fetchers.extend(PyPIFetcher(index) for index in options.indices) - - translator = translator_from_options(options) + fetchers = fetchers_from_options(options) + translator = translator_from_options(interpreter, options) if options.use_wheel: precedence = (WheelPackage, EggPackage, SourcePackage) @@ -322,7 +422,8 @@ def main(): with TraceLogger.env_override(PEX_VERBOSE=options.verbosity): - pex_builder = build_pex(args, options) + with TRACER.timed('Building pex'): + pex_builder = build_pex(args, options) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, v=options.verbosity) diff --git a/pex/http.py b/pex/http.py index ddd49e4f0..38883cd82 100644 --- a/pex/http.py +++ b/pex/http.py @@ -58,7 +58,9 @@ def register(cls, context_impl): def get(cls): for context_class in cls._REGISTRY: try: - return context_class() + context = context_class() + TRACER.log('Constructed %s context %r' % (context.__class__.__name__, context), V=4) + return context except cls.Error: continue raise cls.Error('Could not initialize a request context.') @@ -121,6 +123,11 @@ def content(self, link): encoding = message_from_string(str(fp.headers)).get_content_charset(self.DEFAULT_ENCODING) return fp.read().decode(encoding, 'replace') + def __init__(self, *args, **kw): + TRACER.log('Warning, using a UrllibContext which is known to be flaky.') + TRACER.log('Please build pex with the requests module for more reliable downloads.') + super(UrllibContext, self).__init__(*args, **kw) + Context.register(UrllibContext) @@ -185,6 +192,9 @@ def _create_session(max_retries): return session def __init__(self, session=None, verify=True, max_retries=5): + if requests is None: + raise RuntimeError('requests is not available. Cannot use a RequestsContext.') + self._verify = verify max_retries = int(os.environ.get('PEX_HTTP_RETRIES', max_retries)) diff --git a/pex/version.py b/pex/version.py index 458aba7dc..e3baacf06 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1 +1,4 @@ __version__ = '0.8.7-dev0' + +__setuptools_requirement = 'setuptools>=2.2,<8' +__wheel_requirement = 'wheel>=0.24.0,<0.25.0' diff --git a/setup.py b/setup.py index 1c899d7d6..fe7512a6a 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,11 @@ # This seems to be a fairly standard version file pattern. +# +# Populates the following variables: +# __version__ +# __setuptools_requirement +# __wheel_requirement __version__ = '' version_py_file = os.path.join(os.path.dirname(__file__), 'pex', 'version.py') with open(version_py_file) as version_py: @@ -36,7 +41,7 @@ 'pex.bin', ], install_requires = [ - 'setuptools>=2.2,<8', + __setuptools_requirement, ], tests_require = [ 'mock', @@ -48,10 +53,7 @@ ], entry_points = { 'console_scripts': [ - 'pex = pex.bin.pex:main [whl]', + 'pex = pex.bin.pex:main', ], }, - extras_require = { - 'whl': ['wheel>=0.24.0,<0.25.0'], - }, ) diff --git a/tests/test_http.py b/tests/test_http.py index 1b7c6dfbb..4f42a48cd 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -5,6 +5,7 @@ import pytest from twitter.common.contextutil import temporary_file +from pex.compatibility import PY2 from pex.http import Context, RequestsContext, StreamFilelike, UrllibContext from pex.link import Link @@ -27,6 +28,12 @@ NO_REQUESTS = 'RequestsMock is None or requests is None' +try: + from httplib import HTTPMessage +except ImportError: + from http.client import HTTPMessage + + def make_md5(blob): md5 = hashlib.md5() md5.update(blob) @@ -108,10 +115,14 @@ def __init__(self, data): self.status = 200 self.version = 'HTTP/1.1' self.reason = 'OK' - self.msg = mock.Mock() + if PY2: + self.msg = HTTPMessage(BytesIO('Content-Type: application/x-compressed\r\n')) + else: + self.msg = HTTPMessage() + self.msg.add_header('Content-Type', 'application/x-compressed') def getheaders(self): - return [('Content-Type', 'application/x-compressed')] + return list(self.msg.items()) def isclosed(self): return self.closed