diff --git a/src/python/pants/backend/native/subsystems/conan.py b/src/python/pants/backend/native/subsystems/conan.py index fb8fef89172..ae7d53880bd 100644 --- a/src/python/pants/backend/native/subsystems/conan.py +++ b/src/python/pants/backend/native/subsystems/conan.py @@ -20,6 +20,7 @@ class Conan(PythonToolBase): 'pylint==1.9.3', ] default_entry_point = 'conans.conan' + default_interpreter_constraints = ['CPython>=2.7,<4'] @classmethod def register_options(cls, register): diff --git a/src/python/pants/backend/python/subsystems/python_tool_base.py b/src/python/pants/backend/python/subsystems/python_tool_base.py index 149f671a82c..54b6495e1b0 100644 --- a/src/python/pants/backend/python/subsystems/python_tool_base.py +++ b/src/python/pants/backend/python/subsystems/python_tool_base.py @@ -13,10 +13,16 @@ class PythonToolBase(Subsystem): # Subclasses must set. default_requirements = None default_entry_point = None + # Subclasses need not override. + default_interpreter_constraints = [] @classmethod def register_options(cls, register): super(PythonToolBase, cls).register_options(register) + register('--interpreter-constraints', type=list, advanced=True, fingerprint=True, + default=cls.default_interpreter_constraints, + help='Python interpreter constraints for this tool. An empty list uses the default ' + 'interpreter constraints for the repo.') register('--requirements', type=list, advanced=True, fingerprint=True, default=cls.default_requirements, help='Python requirement strings for the tool.') @@ -24,6 +30,9 @@ def register_options(cls, register): default=cls.default_entry_point, help='The main module for the tool.') + def get_interpreter_constraints(self): + return self.get_options().interpreter_constraints + def get_requirement_specs(self): return self.get_options().requirements diff --git a/src/python/pants/backend/python/tasks/python_tool_prep_base.py b/src/python/pants/backend/python/tasks/python_tool_prep_base.py index 7f0de9db4d6..203da570afd 100644 --- a/src/python/pants/backend/python/tasks/python_tool_prep_base.py +++ b/src/python/pants/backend/python/tasks/python_tool_prep_base.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals import os +from builtins import str from contextlib import contextmanager from pex.pex import PEX @@ -13,6 +14,8 @@ from pants.backend.python.interpreter_cache import PythonInterpreterCache from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.subsystems.pex_build_util import PexBuilderWrapper +from pants.base.build_environment import get_pants_cachedir +from pants.base.hash_utils import stable_json_sha1 from pants.base.workunit import WorkUnitLabel from pants.task.task import Task from pants.util.dirutil import safe_concurrent_creation @@ -23,11 +26,16 @@ class PythonToolInstance(object): def __init__(self, pex_path, interpreter): self._pex = PEX(pex_path, interpreter=interpreter) + self._interpreter = interpreter @property def pex(self): return self._pex + @property + def interpreter(self): + return self._interpreter + def _pretty_cmdline(self, args): return safe_shlex_join(self._pex.cmdline(args)) @@ -63,6 +71,12 @@ def run(self, *args, **kwargs): return cmdline, exit_code +# TODO: This python tool setup ends up eagerly generating each pex for each task in every goal which +# is transitively required by the command-line goals, even for tasks which no-op. This requires each +# pex for each relevant python tool to be buildable on the current host, even if it may never be +# intended to be invoked. Especially given the existing clear separation of concerns into +# PythonToolBase/PythonToolInstance/PythonToolPrepBase, this seems like an extremely ripe use case +# for some v2 rules for free caching and no-op when not required for the command-line goals. class PythonToolPrepBase(Task): """Base class for tasks that resolve a python tool to be invoked out-of-process.""" @@ -97,16 +111,30 @@ def _build_tool_pex(self, tool_subsystem, interpreter, pex_path): pex_builder.set_entry_point(tool_subsystem.get_entry_point()) pex_builder.freeze() + def _generate_fingerprinted_pex_path(self, tool_subsystem, interpreter): + # `tool_subsystem.get_requirement_specs()` is a list, but order shouldn't actually matter. This + # should probably be sorted, but it's possible a user could intentionally tweak order to work + # around a particular requirement resolution resolve-order issue. In practice the lists are + # expected to be mostly static, so we accept the risk of too-fine-grained caching creating lots + # of pexes in the cache dir. + specs_fingerprint = stable_json_sha1(tool_subsystem.get_requirement_specs()) + return os.path.join( + get_pants_cachedir(), + 'python', + str(interpreter.identity), + self.fingerprint, + '{}-{}.pex'.format(tool_subsystem.options_scope, specs_fingerprint), + ) + def execute(self): tool_subsystem = self.tool_subsystem_cls.scoped_instance(self) - pex_name = tool_subsystem.options_scope - pex_path = os.path.join(self.workdir, self.fingerprint, '{}.pex'.format(pex_name)) interpreter_cache = PythonInterpreterCache.global_instance() - interpreter = interpreter_cache.select_interpreter_for_targets([]) + interpreter = min(interpreter_cache.setup(filters=tool_subsystem.get_interpreter_constraints())) + pex_path = self._generate_fingerprinted_pex_path(tool_subsystem, interpreter) if not os.path.exists(pex_path): - with self.context.new_workunit(name='create-{}-pex'.format(pex_name), + with self.context.new_workunit(name='create-{}-pex'.format(tool_subsystem.options_scope), labels=[WorkUnitLabel.PREP]): self._build_tool_pex(tool_subsystem=tool_subsystem, interpreter=interpreter, diff --git a/tests/python/pants_test/backend/python/tasks/test_python_tool.py b/tests/python/pants_test/backend/python/tasks/test_python_tool.py new file mode 100644 index 00000000000..2c030473f10 --- /dev/null +++ b/tests/python/pants_test/backend/python/tasks/test_python_tool.py @@ -0,0 +1,87 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import re + +from pants.backend.python.subsystems.python_tool_base import PythonToolBase +from pants.backend.python.tasks.python_tool_prep_base import PythonToolInstance, PythonToolPrepBase +from pants.task.task import Task +from pants.util.contextutil import temporary_dir +from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase + + +class Tool(PythonToolBase): + options_scope = 'test-tool' + default_requirements = [ + 'pex==1.5.3', + ] + default_entry_point = 'pex.bin.pex:main' + + +class ToolInstance(PythonToolInstance): + pass + + +class ToolPrep(PythonToolPrepBase): + options_scope = 'tool-prep-task' + tool_subsystem_cls = Tool + tool_instance_cls = ToolInstance + + +class ToolTask(Task): + options_scope = 'tool-task' + + @classmethod + def prepare(cls, options, round_manager): + super(ToolTask, cls).prepare(options, round_manager) + round_manager.require_data(ToolPrep.tool_instance_cls) + + def execute(self): + tool_for_pex = self.context.products.get_data(ToolPrep.tool_instance_cls) + stdout, _, exit_code, _ = tool_for_pex.output(['--version']) + assert re.match(r'.*\.pex 1.5.3', stdout) + assert 0 == exit_code + + +class PythonToolPrepTest(PythonTaskTestBase): + + @classmethod + def task_type(cls): + return ToolTask + + def _assert_tool_execution_for_python_version(self, use_py3=True): + scope_string = '3' if use_py3 else '2' + constraint_string = 'CPython>=3' if use_py3 else 'CPython<3' + tool_prep_type = self.synthesize_task_subtype(ToolPrep, 'tp_scope_py{}'.format(scope_string)) + with temporary_dir() as tmp_dir: + context = self.context(for_task_types=[tool_prep_type], for_subsystems=[Tool], options={ + '': { + 'pants_bootstrapdir': tmp_dir, + }, + 'test-tool': { + 'interpreter_constraints': [constraint_string], + }, + }) + tool_prep_task = tool_prep_type(context, os.path.join( + self.pants_workdir, 'tp_py{}'.format(scope_string))) + tool_prep_task.execute() + # Check that the tool can be created and executed successfully. + self.create_task(context).execute() + pex_tool = context.products.get_data(ToolPrep.tool_instance_cls) + # Check that our pex tool wrapper was constructed with the expected interpreter. + self.assertTrue(pex_tool.interpreter.identity.matches(constraint_string)) + return pex_tool + + def test_tool_execution(self): + """Test that python tools are fingerprinted by python interpreter.""" + py3_pex_tool = self._assert_tool_execution_for_python_version(use_py3=True) + py3_pex_tool_path = py3_pex_tool.pex.path() + self.assertTrue(os.path.isdir(py3_pex_tool_path)) + py2_pex_tool = self._assert_tool_execution_for_python_version(use_py3=False) + py2_pex_tool_path = py2_pex_tool.pex.path() + self.assertTrue(os.path.isdir(py2_pex_tool_path)) + self.assertNotEqual(py3_pex_tool_path, py2_pex_tool_path)