From d8f0bd52827ec673a7f07ef818ea5ea740acd908 Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri, 25 Jan 2019 11:30:12 -0800 Subject: [PATCH] remove support for using the pants native toolchain with distutils Extensions in setup.py (#7126) ### Problem Resolves #7016 (see [mirrored `pants-devel` post](https://groups.google.com/forum/#!topic/pants-devel/Y37d0tf4bKo)). This unblocks #6855 and #7046, as well as further work on #7122. Closes #5661, closes #6841. I also made a long, long-overdue [github project for native code](https://github.com/pantsbuild/pants/projects/11). ### Solution - Use the host environment to invoke compilers and linkers as desired in distutils, don't try to inject our own toolchain. See #5661 and #7016 for why this is extremely difficult to maintain. ### Result Further native backend iteration is unblocked. --- build-support/known_py3_pex_failures.txt | 3 +- .../python_distribution/hello/fasthello/BUILD | 25 -- .../hello/fasthello/c_greet.c | 34 --- .../hello/fasthello/cpp_greet.cpp | 34 --- .../hello/fasthello/hello_package/hello.py | 15 -- .../hello/fasthello/main.py | 12 - .../hello/fasthello/setup.py | 26 --- .../hello/setup_requires/BUILD | 34 --- .../setup_requires/hello_package/__init__.py | 0 .../hello/setup_requires/setup.cfg | 2 - .../hello/setup_requires/setup.py | 25 -- .../hello/test_fasthello/BUILD | 14 -- .../hello/test_fasthello/test_fasthello.py | 16 -- .../pants/backend/native/tasks/conan_fetch.py | 5 +- .../python/subsystems/python_native_code.py | 78 +------ .../tasks/build_local_python_distributions.py | 87 ++----- .../python/tasks/resolve_requirements.py | 2 + .../fasthello_with_install_requires/BUILD | 45 ---- .../fasthello_with_install_requires/c_greet.c | 34 --- .../cpp_greet.cpp | 34 --- .../hello_package/__init__.py | 0 .../hello_package/hello.py | 12 - .../fasthello_with_install_requires/setup.py | 28 --- .../hello_with_install_requires/BUILD | 19 ++ .../hello_package/__init__.py | 0 .../hello_package/hello.py | 6 +- .../main.py | 0 .../hello_with_install_requires/setup.py | 13 +- .../example_test/python_distribution/BUILD | 8 + .../python_distribution/test_hello.py | 15 ++ .../pants_test/backend/python/tasks/BUILD | 107 +-------- .../backend/python/tasks/native/BUILD | 63 ++++++ .../python/tasks/{ => native}/test_ctypes.py | 82 +++---- .../{ => native}/test_ctypes_integration.py | 56 ----- .../test_build_local_python_distributions.py | 186 +++++++-------- ..._local_python_distributions_integration.py | 102 +++++++++ .../test_python_distribution_integration.py | 214 ------------------ .../tasks/util/build_local_dists_test_base.py | 127 ++++++++--- 38 files changed, 477 insertions(+), 1086 deletions(-) delete mode 100644 examples/src/python/example/python_distribution/hello/fasthello/BUILD delete mode 100644 examples/src/python/example/python_distribution/hello/fasthello/c_greet.c delete mode 100644 examples/src/python/example/python_distribution/hello/fasthello/cpp_greet.cpp delete mode 100644 examples/src/python/example/python_distribution/hello/fasthello/hello_package/hello.py delete mode 100644 examples/src/python/example/python_distribution/hello/fasthello/main.py delete mode 100644 examples/src/python/example/python_distribution/hello/fasthello/setup.py delete mode 100644 examples/src/python/example/python_distribution/hello/setup_requires/BUILD delete mode 100644 examples/src/python/example/python_distribution/hello/setup_requires/hello_package/__init__.py delete mode 100644 examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg delete mode 100644 examples/src/python/example/python_distribution/hello/setup_requires/setup.py delete mode 100644 examples/tests/python/example/python_distribution/hello/test_fasthello/BUILD delete mode 100644 examples/tests/python/example/python_distribution/hello/test_fasthello/test_fasthello.py delete mode 100644 testprojects/src/python/python_distribution/fasthello_with_install_requires/BUILD delete mode 100644 testprojects/src/python/python_distribution/fasthello_with_install_requires/c_greet.c delete mode 100644 testprojects/src/python/python_distribution/fasthello_with_install_requires/cpp_greet.cpp delete mode 100644 testprojects/src/python/python_distribution/fasthello_with_install_requires/hello_package/__init__.py delete mode 100644 testprojects/src/python/python_distribution/fasthello_with_install_requires/hello_package/hello.py delete mode 100644 testprojects/src/python/python_distribution/fasthello_with_install_requires/setup.py create mode 100644 testprojects/src/python/python_distribution/hello_with_install_requires/BUILD rename {examples/src/python/example/python_distribution/hello/fasthello => testprojects/src/python/python_distribution/hello_with_install_requires}/hello_package/__init__.py (100%) rename {examples/src/python/example/python_distribution/hello/setup_requires => testprojects/src/python/python_distribution/hello_with_install_requires}/hello_package/hello.py (79%) rename testprojects/src/python/python_distribution/{fasthello_with_install_requires => hello_with_install_requires}/main.py (100%) rename examples/src/python/example/python_distribution/hello/setup_requires/main.py => testprojects/src/python/python_distribution/hello_with_install_requires/setup.py (56%) create mode 100644 testprojects/tests/python/example_test/python_distribution/BUILD create mode 100644 testprojects/tests/python/example_test/python_distribution/test_hello.py create mode 100644 tests/python/pants_test/backend/python/tasks/native/BUILD rename tests/python/pants_test/backend/python/tasks/{ => native}/test_ctypes.py (73%) rename tests/python/pants_test/backend/python/tasks/{ => native}/test_ctypes_integration.py (81%) create mode 100644 tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions_integration.py delete mode 100644 tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py diff --git a/build-support/known_py3_pex_failures.txt b/build-support/known_py3_pex_failures.txt index 6755dd3f2a1..4e7f66ab58e 100644 --- a/build-support/known_py3_pex_failures.txt +++ b/build-support/known_py3_pex_failures.txt @@ -1,8 +1,7 @@ tests/python/pants_test/backend/jvm/tasks:scala_repl_integration tests/python/pants_test/backend/project_info/tasks:idea_plugin_integration tests/python/pants_test/backend/python/tasks:integration -tests/python/pants_test/backend/python/tasks:isort_run_integration -tests/python/pants_test/backend/python/tasks:python_native_code_testing_1 +tests/python/pants_test/backend/python:integration tests/python/pants_test/base:exiter_integration tests/python/pants_test/engine/legacy:changed_integration tests/python/pants_test/engine/legacy:console_rule_integration diff --git a/examples/src/python/example/python_distribution/hello/fasthello/BUILD b/examples/src/python/example/python_distribution/hello/fasthello/BUILD deleted file mode 100644 index 8e7ab90bb2f..00000000000 --- a/examples/src/python/example/python_distribution/hello/fasthello/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -# Like Hello world, but built with a python_dist. -# python_dist allows you to use setup.py to depend on C/C++ extensions. - -python_dist( - name='fasthello', - sources=[ - 'c_greet.c', - 'cpp_greet.cpp', - 'hello_package/hello.py', - 'hello_package/__init__.py', - 'setup.py' - ] -) - -python_binary( - name='main', - source='main.py', - dependencies=[ - ':fasthello', - ], - platforms=['current'] -) diff --git a/examples/src/python/example/python_distribution/hello/fasthello/c_greet.c b/examples/src/python/example/python_distribution/hello/fasthello/c_greet.c deleted file mode 100644 index 10837efd043..00000000000 --- a/examples/src/python/example/python_distribution/hello/fasthello/c_greet.c +++ /dev/null @@ -1,34 +0,0 @@ -#include - -static PyObject * c_greet(PyObject *self, PyObject *args) { - return Py_BuildValue("s", "Hello from C!"); -} - -static PyMethodDef Methods[] = { - {"c_greet", c_greet, METH_VARARGS, "A greeting in the C language."}, - {NULL, NULL, 0, NULL} -}; - -#if PY_MAJOR_VERSION >= 3 - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "c_greet", /* m_name */ - NULL, /* m_doc */ - -1, /* m_size */ - Methods, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ - }; -#endif - -#if PY_MAJOR_VERSION >= 3 - PyMODINIT_FUNC PyInit_c_greet(void) { - return PyModule_Create(&moduledef); - } -#else - PyMODINIT_FUNC initc_greet(void) { - (void) Py_InitModule("c_greet", Methods); - } -#endif diff --git a/examples/src/python/example/python_distribution/hello/fasthello/cpp_greet.cpp b/examples/src/python/example/python_distribution/hello/fasthello/cpp_greet.cpp deleted file mode 100644 index 6c0d7db9430..00000000000 --- a/examples/src/python/example/python_distribution/hello/fasthello/cpp_greet.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include - -static PyObject * cpp_greet(PyObject *self, PyObject *args) { - return Py_BuildValue("s", "Hello from C++!"); -} - -static PyMethodDef Methods[] = { - {"cpp_greet", cpp_greet, METH_VARARGS, "A greeting in the C++ language."}, - {NULL, NULL, 0, NULL} -}; - -#if PY_MAJOR_VERSION >= 3 - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "cpp_greet", /* m_name */ - NULL, /* m_doc */ - -1, /* m_size */ - Methods, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ - }; -#endif - -#if PY_MAJOR_VERSION >= 3 - PyMODINIT_FUNC PyInit_cpp_greet(void) { - return PyModule_Create(&moduledef); - } -#else - PyMODINIT_FUNC initcpp_greet(void) { - (void) Py_InitModule("cpp_greet", Methods); - } -#endif diff --git a/examples/src/python/example/python_distribution/hello/fasthello/hello_package/hello.py b/examples/src/python/example/python_distribution/hello/fasthello/hello_package/hello.py deleted file mode 100644 index c73faf08ee7..00000000000 --- a/examples/src/python/example/python_distribution/hello/fasthello/hello_package/hello.py +++ /dev/null @@ -1,15 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import c_greet -import cpp_greet - - -def hello(): - return '\n'.join([ - c_greet.c_greet(), - cpp_greet.cpp_greet(), - ]) diff --git a/examples/src/python/example/python_distribution/hello/fasthello/main.py b/examples/src/python/example/python_distribution/hello/fasthello/main.py deleted file mode 100644 index 2bcf387d513..00000000000 --- a/examples/src/python/example/python_distribution/hello/fasthello/main.py +++ /dev/null @@ -1,12 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -# hello_package is a python module within the fasthello python_distribution -from hello_package import hello - - -if __name__ == '__main__': - print(hello.hello()) diff --git a/examples/src/python/example/python_distribution/hello/fasthello/setup.py b/examples/src/python/example/python_distribution/hello/fasthello/setup.py deleted file mode 100644 index caede584c52..00000000000 --- a/examples/src/python/example/python_distribution/hello/fasthello/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import sys - -from setuptools import setup, find_packages -from distutils.core import Extension - -PY3 = sys.version_info[0] == 3 - -if PY3: - c_module = Extension('c_greet', sources=['c_greet.c']) - cpp_module = Extension('cpp_greet', sources=['cpp_greet.cpp']) -else: - c_module = Extension(b'c_greet', sources=[b'c_greet.c']) - cpp_module = Extension(b'cpp_greet', sources=[b'cpp_greet.cpp']) - -setup( - name='fasthello', - version='1.0.0', - ext_modules=[c_module, cpp_module], - packages=find_packages(), -) diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/BUILD b/examples/src/python/example/python_distribution/hello/setup_requires/BUILD deleted file mode 100644 index ca852ee8104..00000000000 --- a/examples/src/python/example/python_distribution/hello/setup_requires/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -# Like Hello world, but built with a python_dist. -# python_dist allows you to use setup.py to depend on C/C++ extensions. - -python_dist( - name='hello', - sources=[ - 'hello_package/hello.py', - 'hello_package/__init__.py', - 'setup.cfg', - 'setup.py' - ], - setup_requires=[ - 'examples/src/python/example/python_distribution/hello/setup_requires:setup_requires_lib' - ] -) - -python_requirement_library( - name = "setup_requires_lib", - requirements = [ - python_requirement("pycountry==18.5.20"), - ], -) - -python_binary( - name='main', - source='main.py', - dependencies=[ - ':hello', - ], - platforms=['current'] -) diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/__init__.py b/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg b/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg deleted file mode 100644 index 3480374bc2f..00000000000 --- a/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 \ No newline at end of file diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/setup.py b/examples/src/python/example/python_distribution/hello/setup_requires/setup.py deleted file mode 100644 index 3a035a93957..00000000000 --- a/examples/src/python/example/python_distribution/hello/setup_requires/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import os -from setuptools import setup, find_packages - -# We require pycountry with setup_requires argument to this setup script's -# corresponding python_dist. -import pycountry - -# This is for testing purposes so we can assert that setup_requires is functioning -# correctly (because Pants swallows print statements). -if os.getenv('PANTS_TEST_SETUP_REQUIRES', ''): - output = [bytes(pycountry)] - output.extend(os.listdir(os.getenv('PYTHONPATH', ''))) - raise Exception(str(output)) - -setup( - name='hello', - version='1.0.0', - packages=find_packages(), -) diff --git a/examples/tests/python/example/python_distribution/hello/test_fasthello/BUILD b/examples/tests/python/example/python_distribution/hello/test_fasthello/BUILD deleted file mode 100644 index 0b7a6ba9fb9..00000000000 --- a/examples/tests/python/example/python_distribution/hello/test_fasthello/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -# Example of defining a test target that depends on a python_dist target. - -python_tests( - name='fasthello', - sources=[ - 'test_fasthello.py' - ], - dependencies=[ - 'examples/src/python/example/python_distribution/hello/fasthello:fasthello' - ] -) diff --git a/examples/tests/python/example/python_distribution/hello/test_fasthello/test_fasthello.py b/examples/tests/python/example/python_distribution/hello/test_fasthello/test_fasthello.py deleted file mode 100644 index 5e0d6af3518..00000000000 --- a/examples/tests/python/example/python_distribution/hello/test_fasthello/test_fasthello.py +++ /dev/null @@ -1,16 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -# hello_package is a python module within the fasthello python_distribution. -from hello_package import hello - - -# Example of writing a test that depends on a python_dist target. -def test_fasthello(): - assert hello.hello() == '\n'.join([ - 'Hello from C!', - 'Hello from C++!', - ]) diff --git a/src/python/pants/backend/native/tasks/conan_fetch.py b/src/python/pants/backend/native/tasks/conan_fetch.py index 67b254def26..84d9b9951ad 100644 --- a/src/python/pants/backend/native/tasks/conan_fetch.py +++ b/src/python/pants/backend/native/tasks/conan_fetch.py @@ -133,8 +133,9 @@ def execute_codegen(self, target, target_workdir): .format(cmdline, env, exit_code), exit_code=exit_code) - # Read the stdout from the read-write buffer, from the beginning of the output. - conan_install_stdout = workunit.output('stdout').read_from(0) + # Read the stdout from the read-write buffer, from the beginning of the output, and convert + # to unicode. + conan_install_stdout = workunit.output('stdout').read_from(0).decode('utf-8') pkg_sha = conan_requirement.parse_conan_stdout_for_pkg_sha(conan_install_stdout) installed_data_dir = os.path.join( diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index 8ca5e5cbac1..1b7dc702905 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -4,15 +4,10 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import os from builtins import str from collections import defaultdict -from pex.pex import PEX - -from pants.backend.native.config.environment import CppToolchain, CToolchain, Platform from pants.backend.native.subsystems.native_toolchain import NativeToolchain -from pants.backend.native.subsystems.xcode_cli_tools import MIN_OSX_VERSION_ARG from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.subsystems.python_setup import PythonSetup @@ -22,8 +17,7 @@ from pants.binaries.executable_pex_tool import ExecutablePexTool from pants.subsystem.subsystem import Subsystem from pants.util.memo import memoized_property -from pants.util.objects import SubclassesOf, datatype -from pants.util.strutil import create_path_env_var +from pants.util.objects import SubclassesOf class PythonNativeCode(Subsystem): @@ -65,14 +59,11 @@ def _python_setup(self): def pydist_has_native_sources(self, target): return target.has_sources(extension=tuple(self._native_source_extensions)) - def native_target_has_native_sources(self, target): - return target.has_sources() - @memoized_property def _native_target_matchers(self): return { SubclassesOf(PythonDistribution): self.pydist_has_native_sources, - SubclassesOf(NativeLibrary): self.native_target_has_native_sources, + SubclassesOf(NativeLibrary): NativeLibrary.produces_ctypes_native_library, } def _any_targets_have_native_sources(self, targets): @@ -158,68 +149,3 @@ def base_requirements(self): PythonRequirement('setuptools=={}'.format(self.python_setup.setuptools_version)), PythonRequirement('wheel=={}'.format(self.python_setup.wheel_version)), ] - - -class SetupPyNativeTools(datatype([ - ('c_toolchain', CToolchain), - ('cpp_toolchain', CppToolchain), - ('platform', Platform), -])): - """The native tools needed for a setup.py invocation. - - This class exists because `SetupPyExecutionEnvironment` is created manually, one per target. - """ - - -# TODO: It might be pretty useful to have an Optional TypeConstraint. -class SetupPyExecutionEnvironment(datatype([ - # If None, don't set PYTHONPATH in the setup.py environment. - ('setup_requires_pex', PEX), - # If None, don't execute in the toolchain environment. - 'setup_py_native_tools', -])): - - _SHARED_CMDLINE_ARGS = { - 'darwin': lambda: [ - MIN_OSX_VERSION_ARG, - '-Wl,-dylib', - '-undefined', - 'dynamic_lookup', - ], - 'linux': lambda: ['-shared'], - } - - def as_environment(self): - ret = {} - - # TODO(#5951): the below is a lot of error-prone repeated logic -- we need a way to compose - # executables more hygienically. We should probably be composing each datatype's members, and - # only creating an environment at the very end. - native_tools = self.setup_py_native_tools - if native_tools: - # An as_tuple() method for datatypes could make this destructuring cleaner! Alternatively, - # constructing this environment could be done more compositionally instead of requiring all of - # these disparate fields together at once. - c_toolchain = native_tools.c_toolchain - c_compiler = c_toolchain.c_compiler - c_linker = c_toolchain.c_linker - - cpp_toolchain = native_tools.cpp_toolchain - cpp_compiler = cpp_toolchain.cpp_compiler - cpp_linker = cpp_toolchain.cpp_linker - - all_path_entries = ( - c_compiler.path_entries + - c_linker.path_entries + - cpp_compiler.path_entries + - cpp_linker.path_entries) - # TODO(#6273): We prepend our toolchain to the PATH instead of overwriting it -- we need - # better control of the distutils compilation environment if we want to actually isolate the - # PATH (distutils does lots of sneaky things). - ret['PATH'] = create_path_env_var(all_path_entries, env=os.environ.copy(), prepend=True) - - # GCC will output smart quotes in a variety of situations (leading to decoding errors - # downstream) unless we set this environment variable. - ret['LC_ALL'] = 'C' - - return ret diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index 52e2057fb68..93bf53b5e6f 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -12,17 +12,12 @@ from pex import pep425tags from pex.interpreter import PythonInterpreter -from pants.backend.native.config.environment import CppToolchain, CToolchain, Platform -from pants.backend.native.subsystems.native_build_settings import NativeBuildSettings -from pants.backend.native.subsystems.native_toolchain import ToolchainVariantRequest from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.native.tasks.link_shared_libraries import SharedLibrary from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.subsystems.pex_build_util import is_local_python_dist from pants.backend.python.subsystems.python_native_code import (BuildSetupRequiresPex, - PythonNativeCode, - SetupPyExecutionEnvironment, - SetupPyNativeTools) + PythonNativeCode) from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.base.build_environment import get_buildroot from pants.base.exceptions import TargetDefinitionException, TaskError @@ -32,10 +27,11 @@ from pants.util.collections import assert_single_element from pants.util.contextutil import pushd from pants.util.dirutil import safe_mkdir_for, split_basename_and_dirname -from pants.util.memo import memoized_classproperty, memoized_property +from pants.util.memo import memoized_property from pants.util.strutil import safe_shlex_join +# TODO: make this a SimpleCodegenTask!!! class BuildLocalPythonDistributions(Task): """Create python distributions (.whl) from python_dist targets.""" @@ -73,52 +69,19 @@ def implementation_version(cls): def subsystem_dependencies(cls): return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + ( BuildSetupRequiresPex.scoped(cls), - NativeBuildSettings, PythonNativeCode.scoped(cls), ) class BuildLocalPythonDistributionsError(TaskError): pass - @memoized_classproperty - def _platform(cls): - return Platform.create() - @memoized_property def _python_native_code_settings(self): return PythonNativeCode.scoped_instance(self) - @memoized_property - def _native_toolchain(self): - return self._python_native_code_settings.native_toolchain - - @memoized_property - def _toolchain_variant_request(self): - return ToolchainVariantRequest( - toolchain=self._native_toolchain, - variant=self._native_build_settings.toolchain_variant) - - @memoized_property - def _native_build_settings(self): - return NativeBuildSettings.global_instance() - @memoized_property def _build_setup_requires_pex_settings(self): return BuildSetupRequiresPex.scoped_instance(self) - # TODO(#5869): delete this and get Subsystems from options, when that is possible. - def _request_single(self, product, subject): - # NB: This is not supposed to be exposed to Tasks yet -- see #4769 to track the status of - # exposing v2 products in v1 tasks. - return self.context._scheduler.product_request(product, [subject])[0] - - @memoized_property - def _c_toolchain(self): - return self._request_single(CToolchain, self._toolchain_variant_request) - - @memoized_property - def _cpp_toolchain(self): - return self._request_single(CppToolchain, self._toolchain_variant_request) - # TODO: This should probably be made into an @classproperty (see PR #5901). @property def cache_target_dirs(self): @@ -228,20 +191,14 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t all_native_artifacts = self._add_artifacts( dist_output_dir, shared_libs_product, native_artifact_deps) - is_platform_specific = False - native_tools = None - if self._python_native_code_settings.pydist_has_native_sources(dist_target): - # We add the native tools if we need to compile code belonging to this python_dist() target. - # TODO: test this branch somehow! - native_tools = SetupPyNativeTools( - c_toolchain=self._c_toolchain, - cpp_toolchain=self._cpp_toolchain, - platform=self._platform) - # Native code in this python_dist() target requires marking the dist as platform-specific. - is_platform_specific = True - elif len(all_native_artifacts) > 0: + # TODO: remove the triplication all of this validation across _get_native_artifact_deps(), + # check_build_for_current_platform_only(), and len(all_native_artifacts) > 0! + is_platform_specific = ( # We are including a platform-specific shared lib in this dist, so mark it as such. - is_platform_specific = True + len(all_native_artifacts) > 0 + or self._python_native_code_settings.check_build_for_current_platform_only( + # NB: This doesn't reach into transitive dependencies, but that doesn't matter currently. + [dist_target] + dist_target.dependencies)) versioned_target_fingerprint = versioned_target.cache_key.hash @@ -255,20 +212,15 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t setup_reqs_pex_path = os.path.join( setup_requires_dir, 'setup-requires-{}.pex'.format(versioned_target_fingerprint)) - extra_reqs = list(setup_reqs_to_resolve or []) setup_requires_pex = self._build_setup_requires_pex_settings.bootstrap( - interpreter, setup_reqs_pex_path, extra_reqs=extra_reqs) + interpreter, setup_reqs_pex_path, extra_reqs=setup_reqs_to_resolve) self.context.log.debug('Using pex file as setup.py interpreter: {}' .format(setup_requires_pex)) - setup_py_execution_environment = SetupPyExecutionEnvironment( - setup_requires_pex=setup_requires_pex, - setup_py_native_tools=native_tools) - self._create_dist( dist_target, dist_output_dir, - setup_py_execution_environment, + setup_requires_pex, versioned_target_fingerprint, is_platform_specific) @@ -301,7 +253,7 @@ def _generate_snapshot_bdist_wheel_argv(self, snapshot_fingerprint, is_platform_ def _create_dist(self, dist_tgt, dist_target_dir, - setup_py_execution_environment, + setup_requires_pex, snapshot_fingerprint, is_platform_specific): """Create a .whl file for the specified python_distribution target.""" @@ -310,28 +262,23 @@ def _create_dist(self, setup_py_snapshot_version_argv = self._generate_snapshot_bdist_wheel_argv( snapshot_fingerprint, is_platform_specific) - setup_requires_pex = setup_py_execution_environment.setup_requires_pex - setup_py_env = setup_py_execution_environment.as_environment() - cmd = safe_shlex_join(setup_requires_pex.cmdline(setup_py_snapshot_version_argv)) with self.context.new_workunit('setup.py', cmd=cmd, labels=[WorkUnitLabel.TOOL]) as workunit: with pushd(dist_target_dir): result = setup_requires_pex.run(args=setup_py_snapshot_version_argv, - env=setup_py_env, stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) if result != 0: raise self.BuildLocalPythonDistributionsError( "Installation of python distribution from target {target} into directory {into_dir} " "failed (return value of run() was: {rc!r}).\n" - "The chosen interpreter was: {interpreter}.\n" - "The execution environment was: {env}.\n" + "The pex with any requirements is located at: {interpreter}.\n" + "The host system's compiler and linker were used.\n" "The setup command was: {command}." .format(target=dist_tgt, into_dir=dist_target_dir, rc=result, interpreter=setup_requires_pex.path(), - env=setup_py_env, command=setup_py_snapshot_version_argv)) def _inject_synthetic_dist_requirements(self, dist, req_lib_addr): @@ -355,7 +302,9 @@ def _get_whl_from_dir(cls, install_dir): dists = glob.glob(os.path.join(dist_dir, '*.whl')) if len(dists) == 0: raise cls.BuildLocalPythonDistributionsError( - 'No distributions were produced by python_create_distribution task.') + 'No distributions were produced by python_create_distribution task.\n' + 'dist_dir: {}, install_dir: {}' + .format(dist_dir, install_dir)) if len(dists) > 1: # TODO: is this ever going to happen? raise cls.BuildLocalPythonDistributionsError('Ambiguous local python distributions found: {}' diff --git a/src/python/pants/backend/python/tasks/resolve_requirements.py b/src/python/pants/backend/python/tasks/resolve_requirements.py index eadde759a90..ea8b74fb9bd 100644 --- a/src/python/pants/backend/python/tasks/resolve_requirements.py +++ b/src/python/pants/backend/python/tasks/resolve_requirements.py @@ -14,6 +14,8 @@ class ResolveRequirements(ResolveRequirementsTaskBase): """Resolve external Python requirements.""" REQUIREMENTS_PEX = 'python_requirements_pex' + options_scope = 'resolve-requirements' + @classmethod def product_types(cls): return [cls.REQUIREMENTS_PEX] diff --git a/testprojects/src/python/python_distribution/fasthello_with_install_requires/BUILD b/testprojects/src/python/python_distribution/fasthello_with_install_requires/BUILD deleted file mode 100644 index ebb16c87c8c..00000000000 --- a/testprojects/src/python/python_distribution/fasthello_with_install_requires/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - - -python_dist( - name='fasthello', - sources=[ - 'c_greet.c', - 'cpp_greet.cpp', - 'hello_package/hello.py', - 'hello_package/__init__.py', - 'setup.py' - ] -) - -python_binary( - name='main_with_no_conflict', - source='main.py', - dependencies=[ - ':fasthello', - ], - platforms=['current'] -) - -python_binary( - name='main_with_conflicting_dep', - source='main.py', - dependencies=[ - ':fasthello', - ':pycountry' - ], - platforms=['current'] -) - -python_requirement_library( - name='pycountry', - requirements=[ - python_requirement('pycountry==17.9.23'), - ] -) - -python_binary( - name='main_with_no_pycountry', - source='main.py' -) diff --git a/testprojects/src/python/python_distribution/fasthello_with_install_requires/c_greet.c b/testprojects/src/python/python_distribution/fasthello_with_install_requires/c_greet.c deleted file mode 100644 index 10837efd043..00000000000 --- a/testprojects/src/python/python_distribution/fasthello_with_install_requires/c_greet.c +++ /dev/null @@ -1,34 +0,0 @@ -#include - -static PyObject * c_greet(PyObject *self, PyObject *args) { - return Py_BuildValue("s", "Hello from C!"); -} - -static PyMethodDef Methods[] = { - {"c_greet", c_greet, METH_VARARGS, "A greeting in the C language."}, - {NULL, NULL, 0, NULL} -}; - -#if PY_MAJOR_VERSION >= 3 - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "c_greet", /* m_name */ - NULL, /* m_doc */ - -1, /* m_size */ - Methods, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ - }; -#endif - -#if PY_MAJOR_VERSION >= 3 - PyMODINIT_FUNC PyInit_c_greet(void) { - return PyModule_Create(&moduledef); - } -#else - PyMODINIT_FUNC initc_greet(void) { - (void) Py_InitModule("c_greet", Methods); - } -#endif diff --git a/testprojects/src/python/python_distribution/fasthello_with_install_requires/cpp_greet.cpp b/testprojects/src/python/python_distribution/fasthello_with_install_requires/cpp_greet.cpp deleted file mode 100644 index 6c0d7db9430..00000000000 --- a/testprojects/src/python/python_distribution/fasthello_with_install_requires/cpp_greet.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include - -static PyObject * cpp_greet(PyObject *self, PyObject *args) { - return Py_BuildValue("s", "Hello from C++!"); -} - -static PyMethodDef Methods[] = { - {"cpp_greet", cpp_greet, METH_VARARGS, "A greeting in the C++ language."}, - {NULL, NULL, 0, NULL} -}; - -#if PY_MAJOR_VERSION >= 3 - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "cpp_greet", /* m_name */ - NULL, /* m_doc */ - -1, /* m_size */ - Methods, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ - }; -#endif - -#if PY_MAJOR_VERSION >= 3 - PyMODINIT_FUNC PyInit_cpp_greet(void) { - return PyModule_Create(&moduledef); - } -#else - PyMODINIT_FUNC initcpp_greet(void) { - (void) Py_InitModule("cpp_greet", Methods); - } -#endif diff --git a/testprojects/src/python/python_distribution/fasthello_with_install_requires/hello_package/__init__.py b/testprojects/src/python/python_distribution/fasthello_with_install_requires/hello_package/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/python/python_distribution/fasthello_with_install_requires/hello_package/hello.py b/testprojects/src/python/python_distribution/fasthello_with_install_requires/hello_package/hello.py deleted file mode 100644 index 051ba5de472..00000000000 --- a/testprojects/src/python/python_distribution/fasthello_with_install_requires/hello_package/hello.py +++ /dev/null @@ -1,12 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import c_greet -import cpp_greet - -def hello(): - print(c_greet.c_greet()) - print(cpp_greet.cpp_greet()) diff --git a/testprojects/src/python/python_distribution/fasthello_with_install_requires/setup.py b/testprojects/src/python/python_distribution/fasthello_with_install_requires/setup.py deleted file mode 100644 index 7ab7c7c884b..00000000000 --- a/testprojects/src/python/python_distribution/fasthello_with_install_requires/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import sys - -from setuptools import setup, find_packages -from distutils.core import Extension - - -PY3 = sys.version_info[0] == 3 - -if PY3: - c_module = Extension('c_greet', sources=['c_greet.c']) - cpp_module = Extension('cpp_greet', sources=['cpp_greet.cpp']) -else: - c_module = Extension(b'c_greet', sources=[b'c_greet.c']) - cpp_module = Extension(b'cpp_greet', sources=[b'cpp_greet.cpp']) - -setup( - name='fasthello_test', - version='1.0.0', - ext_modules=[c_module, cpp_module], - packages=find_packages(), - install_requires=['pycountry==17.1.2'] -) diff --git a/testprojects/src/python/python_distribution/hello_with_install_requires/BUILD b/testprojects/src/python/python_distribution/hello_with_install_requires/BUILD new file mode 100644 index 00000000000..56680965636 --- /dev/null +++ b/testprojects/src/python/python_distribution/hello_with_install_requires/BUILD @@ -0,0 +1,19 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_dist( + sources=[ + 'hello_package/hello.py', + 'hello_package/__init__.py', + 'setup.py' + ] +) + +python_binary( + name='main_with_no_conflict', + source='main.py', + dependencies=[ + ':hello_with_install_requires', + ], + platforms=['current'] +) diff --git a/examples/src/python/example/python_distribution/hello/fasthello/hello_package/__init__.py b/testprojects/src/python/python_distribution/hello_with_install_requires/hello_package/__init__.py similarity index 100% rename from examples/src/python/example/python_distribution/hello/fasthello/hello_package/__init__.py rename to testprojects/src/python/python_distribution/hello_with_install_requires/hello_package/__init__.py diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/hello.py b/testprojects/src/python/python_distribution/hello_with_install_requires/hello_package/hello.py similarity index 79% rename from examples/src/python/example/python_distribution/hello/setup_requires/hello_package/hello.py rename to testprojects/src/python/python_distribution/hello_with_install_requires/hello_package/hello.py index d24910327fc..695003acb5c 100644 --- a/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/hello.py +++ b/testprojects/src/python/python_distribution/hello_with_install_requires/hello_package/hello.py @@ -5,5 +5,9 @@ from __future__ import absolute_import, division, print_function, unicode_literals +def hello_string(): + return 'hello!' + + def hello(): - return 'Hello, world!' + print(hello_string()) diff --git a/testprojects/src/python/python_distribution/fasthello_with_install_requires/main.py b/testprojects/src/python/python_distribution/hello_with_install_requires/main.py similarity index 100% rename from testprojects/src/python/python_distribution/fasthello_with_install_requires/main.py rename to testprojects/src/python/python_distribution/hello_with_install_requires/main.py diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/main.py b/testprojects/src/python/python_distribution/hello_with_install_requires/setup.py similarity index 56% rename from examples/src/python/example/python_distribution/hello/setup_requires/main.py rename to testprojects/src/python/python_distribution/hello_with_install_requires/setup.py index cf04ba357b7..b453096a408 100644 --- a/examples/src/python/example/python_distribution/hello/setup_requires/main.py +++ b/testprojects/src/python/python_distribution/hello_with_install_requires/setup.py @@ -4,8 +4,11 @@ from __future__ import absolute_import, division, print_function, unicode_literals -from hello_package import hello - - -if __name__ == '__main__': - print(hello.hello()) +from setuptools import setup, find_packages + +setup( + name='hello_with_install_requires', + version='1.0.0', + packages=find_packages(), + install_requires=['pycountry==17.1.2'] +) diff --git a/testprojects/tests/python/example_test/python_distribution/BUILD b/testprojects/tests/python/example_test/python_distribution/BUILD new file mode 100644 index 00000000000..d118ca44311 --- /dev/null +++ b/testprojects/tests/python/example_test/python_distribution/BUILD @@ -0,0 +1,8 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_tests( + dependencies=[ + 'testprojects/src/python/python_distribution/hello_with_install_requires', + ], +) diff --git a/testprojects/tests/python/example_test/python_distribution/test_hello.py b/testprojects/tests/python/example_test/python_distribution/test_hello.py new file mode 100644 index 00000000000..ce206f9b69d --- /dev/null +++ b/testprojects/tests/python/example_test/python_distribution/test_hello.py @@ -0,0 +1,15 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import unittest + +from hello_package import hello + + +class HelloTest(unittest.TestCase): + + def test_hello_import(self): + self.assertEqual('hello!', hello.hello_string()) diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index 9d2f46b3c5e..072b3380e38 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -22,85 +22,8 @@ python_library( ] ) -python_native_code_test_files = [ - 'test_build_local_python_distributions.py', - 'test_python_distribution_integration.py', - 'test_ctypes.py', - 'test_ctypes_integration.py', -] - -python_tests( - name='python_native_code_testing_3', - sources=[ - python_native_code_test_files[0], - ], - dependencies=[ - '3rdparty/python:future', - '3rdparty/python/twitter/commons:twitter.common.collections', - 'src/python/pants/backend/native', - 'src/python/pants/backend/native/targets', - 'src/python/pants/backend/python:plugin', - 'src/python/pants/backend/python/targets', - 'src/python/pants/backend/python/tasks', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:int-test', - 'tests/python/pants_test/backend/python/tasks/util', - 'tests/python/pants_test/engine:scheduler_test_base', - 'tests/python/pants_test/testutils:py2_compat', - ], - tags={'platform_specific_behavior', 'integration'}, - timeout=2400, -) - -python_tests( - name='python_native_code_testing_2', - sources=[ - python_native_code_test_files[1], - ], - dependencies=[ - '3rdparty/python:future', - '3rdparty/python/twitter/commons:twitter.common.collections', - 'src/python/pants/backend/native', - 'src/python/pants/backend/native/targets', - 'src/python/pants/backend/python:plugin', - 'src/python/pants/backend/python/targets', - 'src/python/pants/backend/python/tasks', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:int-test', - 'tests/python/pants_test/backend/python/tasks/util', - 'tests/python/pants_test/engine:scheduler_test_base', - 'tests/python/pants_test/testutils:py2_compat', - ], - tags={'platform_specific_behavior', 'integration'}, - timeout=2400, -) - python_tests( - name='python_native_code_testing_1', - sources=python_native_code_test_files[2:], - dependencies=[ - '3rdparty/python:future', - '3rdparty/python/twitter/commons:twitter.common.collections', - 'src/python/pants/backend/native', - 'src/python/pants/backend/native/targets', - 'src/python/pants/backend/python:plugin', - 'src/python/pants/backend/python/targets', - 'src/python/pants/backend/python/tasks', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:int-test', - 'tests/python/pants_test/backend/python/tasks/util', - 'tests/python/pants_test/engine:scheduler_test_base', - 'tests/python/pants_test/testutils:py2_compat', - ], - tags={'platform_specific_behavior', 'integration'}, - timeout=2400, -) - -python_tests( - sources=globs('test_*.py', exclude=([globs('*_integration.py')] + python_native_code_test_files)), + sources=globs('test_*.py', exclude=([globs('*_integration.py')])), dependencies=[ '3rdparty/python/twitter/commons:twitter.common.collections', '3rdparty/python/twitter/commons:twitter.common.dirutil', @@ -128,6 +51,7 @@ python_tests( 'src/python/pants/util:process_handler', 'src/python/pants/util:py2_compat', 'tests/python/pants_test/backend/python:interpreter_selection_utils', + 'tests/python/pants_test/backend/python/tasks/util', 'tests/python/pants_test/engine:scheduler_test_base', 'tests/python/pants_test/subsystem:subsystem_utils', 'tests/python/pants_test/tasks:task_test_base', @@ -139,7 +63,7 @@ python_tests( python_tests( name='integration', - sources=globs('*_integration.py', exclude=python_native_code_test_files), + sources=globs('*_integration.py'), dependencies=[ '3rdparty/python:pex', 'src/python/pants/base:build_environment', @@ -154,28 +78,3 @@ python_tests( tags={'integration'}, timeout=2400 ) - -python_tests( - name='isort_run_integration', - sources=['test_isort_run_integration.py'], - dependencies=[ - 'tests/python/pants_test:int-test', - 'tests/python/pants_test/backend/python/tasks:python_task_test_base', - ], - tags={'integration'}, -) - -python_tests( - name='isort_run', - sources=['test_isort_run.py'], - dependencies=[ - '3rdparty/python:future', - 'src/python/pants/util:dirutil', - 'tests/python/pants_test/backend/python/tasks:python_task_test_base', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:test_base', - 'tests/python/pants_test/subsystem:subsystem_utils', - 'tests/python/pants_test:task_test_base', - ], - timeout=300 -) diff --git a/tests/python/pants_test/backend/python/tasks/native/BUILD b/tests/python/pants_test/backend/python/tasks/native/BUILD new file mode 100644 index 00000000000..7bbccd25921 --- /dev/null +++ b/tests/python/pants_test/backend/python/tasks/native/BUILD @@ -0,0 +1,63 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_tests( + sources=globs('test_*.py', exclude=[globs('*_integration.py')]), + dependencies=[ + '3rdparty/python/twitter/commons:twitter.common.collections', + '3rdparty/python/twitter/commons:twitter.common.dirutil', + '3rdparty/python:coverage', + '3rdparty/python:future', + '3rdparty/python:mock', + '3rdparty/python:pex', + 'src/python/pants/backend/native', + 'src/python/pants/backend/python/subsystems', + 'src/python/pants/backend/python/targets', + 'src/python/pants/backend/python/tasks', + 'src/python/pants/backend/python:interpreter_cache', + 'src/python/pants/backend/python:plugin', + 'src/python/pants/backend/python:python_artifact', + 'src/python/pants/backend/python:python_requirement', + 'src/python/pants/base:build_root', + 'src/python/pants/base:exceptions', + 'src/python/pants/base:run_info', + 'src/python/pants/build_graph', + 'src/python/pants/fs', + 'src/python/pants/source', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', + 'src/python/pants/util:py2_compat', + 'tests/python/pants_test/backend/python:interpreter_selection_utils', + 'tests/python/pants_test/backend/python/tasks:python_task_test_base', + 'tests/python/pants_test/backend/python/tasks/util', + 'tests/python/pants_test/engine:scheduler_test_base', + 'tests/python/pants_test/subsystem:subsystem_utils', + 'tests/python/pants_test/tasks:task_test_base', + 'tests/python/pants_test:task_test_base', + ], + tags={'platform_specific_behavior'}, + timeout=600, +) + +python_tests( + name='integration', + sources=globs('*_integration.py'), + dependencies=[ + '3rdparty/python:future', + '3rdparty/python/twitter/commons:twitter.common.collections', + 'src/python/pants/backend/native', + 'src/python/pants/backend/native/targets', + 'src/python/pants/backend/python:plugin', + 'src/python/pants/backend/python/targets', + 'src/python/pants/backend/python/tasks', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', + 'tests/python/pants_test:int-test', + 'tests/python/pants_test/backend/python/tasks/util', + 'tests/python/pants_test/engine:scheduler_test_base', + 'tests/python/pants_test/testutils:py2_compat', + ], + tags={'platform_specific_behavior', 'integration'}, + timeout=2400, +) diff --git a/tests/python/pants_test/backend/python/tasks/test_ctypes.py b/tests/python/pants_test/backend/python/tasks/native/test_ctypes.py similarity index 73% rename from tests/python/pants_test/backend/python/tasks/test_ctypes.py rename to tests/python/pants_test/backend/python/tasks/native/test_ctypes.py index 24ad01886f8..d1b014a45ea 100644 --- a/tests/python/pants_test/backend/python/tasks/test_ctypes.py +++ b/tests/python/pants_test/backend/python/tasks/native/test_ctypes.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals from builtins import str -from textwrap import dedent from twitter.common.collections import OrderedDict @@ -15,6 +14,7 @@ from pants.backend.native.tasks.cpp_compile import CppCompile from pants.backend.native.tasks.link_shared_libraries import LinkSharedLibraries from pants.backend.python.targets.python_distribution import PythonDistribution +from pants.util.meta import classproperty from pants_test.backend.python.tasks.python_task_test_base import check_wheel_platform_matches_host from pants_test.backend.python.tasks.util.build_local_dists_test_base import \ BuildLocalPythonDistributionsTestBase @@ -22,11 +22,13 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTestBase): - _extra_relevant_task_types = ([ - CCompile, - CppCompile, - LinkSharedLibraries, - ] + BuildLocalPythonDistributionsTestBase._extra_relevant_task_types) + @classproperty + def _run_before_task_types(cls): + return [ + CCompile, + CppCompile, + LinkSharedLibraries, + ] + super(TestBuildLocalDistsWithCtypesNativeSources, cls)._run_before_task_types _dist_specs = OrderedDict([ @@ -36,14 +38,14 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTe 'ctypes_native_library': NativeArtifact(lib_name='c-math-lib'), 'sources': ['c_math_lib.c', 'c_math_lib.h'], 'filemap': { - 'src/python/plat_specific_c_dist/c_math_lib.c': dedent(""" - #include "c_math_lib.h" - int add_two(int x) { return x + 2; } -"""), - 'src/python/plat_specific_c_dist/c_math_lib.h': dedent(""" - int add_two(int); -"""), - } + 'c_math_lib.c': """\ +#include "c_math_lib.h" +int add_two(int x) { return x + 2; } + """, + 'c_math_lib.h': """\ +int add_two(int); + """, + }, }), ('src/python/plat_specific_c_dist:plat_specific_ctypes_c_dist', { @@ -52,17 +54,17 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTe 'sources': ['__init__.py', 'setup.py'], 'dependencies': ['src/python/plat_specific_c_dist:ctypes_c_library'], 'filemap': { - 'src/python/plat_specific_c_dist/__init__.py': '', - 'src/python/plat_specific_c_dist/setup.py': dedent(""" - from setuptools import setup, find_packages - setup( - name='platform_specific_ctypes_c_dist', - version='0.0.0', - packages=find_packages(), - data_files=[('', ['libc-math-lib.so'])], - ) - """), - } + '__init__.py': '', + 'setup.py': """\ +from setuptools import setup, find_packages +setup( + name='platform_specific_ctypes_c_dist', + version='0.0.0', + packages=find_packages(), + data_files=[('', ['libc-math-lib.so'])], +) + """, + }, }), ('src/python/plat_specific_cpp_dist:ctypes_cpp_library', { @@ -71,13 +73,13 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTe 'ctypes_native_library': NativeArtifact(lib_name='cpp-math-lib'), 'sources': ['cpp_math_lib.cpp', 'cpp_math_lib.hpp'], 'filemap': { - 'src/python/plat_specific_cpp_dist/cpp_math_lib.cpp': dedent(""" + 'cpp_math_lib.cpp': """\ #include "cpp_math_lib.hpp" int add_two(int x) { return (x++) + 1; } -"""), - 'src/python/plat_specific_cpp_dist/cpp_math_lib.hpp': dedent(""" + """, + 'cpp_math_lib.hpp': """\ int add_two(int); -"""), + """, }, }), @@ -87,17 +89,17 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTe 'sources': ['__init__.py', 'setup.py'], 'dependencies': ['src/python/plat_specific_cpp_dist:ctypes_cpp_library'], 'filemap': { - 'src/python/plat_specific_cpp_dist/__init__.py': '', - 'src/python/plat_specific_cpp_dist/setup.py': dedent(""" - from setuptools import setup, find_packages - setup( - name='platform_specific_ctypes_cpp_dist', - version='0.0.0', - packages=find_packages(), - data_files=[('', ['libcpp-math-lib.so'])], - ) - """), - } + '__init__.py': '', + 'setup.py': """\ +from setuptools import setup, find_packages +setup( + name='platform_specific_ctypes_cpp_dist', + version='0.0.0', + packages=find_packages(), + data_files=[('', ['libcpp-math-lib.so'])], +) + """, + }, }), ]) diff --git a/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py b/tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py similarity index 81% rename from tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py rename to tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py index 4fbefc86535..f8ef8d2e0cc 100644 --- a/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py +++ b/tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py @@ -38,12 +38,6 @@ class CTypesIntegrationTest(PantsRunIntegrationTest): 'testprojects/src/python/python_distribution/ctypes_with_extra_compiler_flags:bin' ) - def test_ctypes_run(self): - pants_run = self.run_pants(command=['-q', 'run', self._binary_target]) - self.assert_success(pants_run) - - self.assertEqual('x=3, f(x)=17\n', pants_run.stdout_data) - def test_ctypes_binary_creation(self): """Create a python_binary() with all native toolchain variants, and test the result.""" # TODO: this pattern could be made more ergonomic for `enum()`, along with exhaustiveness @@ -116,56 +110,6 @@ def _assert_ctypes_binary_creation(self, toolchain_variant): binary_run_output = invoke_pex_for_output(pex) self.assertEqual(b'x=3, f(x)=17\n', binary_run_output) - def test_invalidation_ctypes(self): - """Test that the current version of a python_dist() is resolved after modifying its sources.""" - with temporary_dir() as tmp_dir: - with self.mock_buildroot( - dirs_to_copy=[self._binary_target_dir]) as buildroot, buildroot.pushd(): - - def run_target(goal): - return self.run_pants_with_workdir( - command=[goal, self._binary_target], - workdir=os.path.join(buildroot.new_buildroot, '.pants.d'), - build_root=buildroot.new_buildroot, - config={ - GLOBAL_SCOPE_CONFIG_SECTION: { - 'pants_distdir': tmp_dir, - }, - }, - ) - - output_pex = os.path.join(tmp_dir, 'bin.pex') - - initial_result_message = 'x=3, f(x)=17' - - unmodified_pants_run = run_target('run') - self.assert_success(unmodified_pants_run) - self.assertIn(initial_result_message, unmodified_pants_run.stdout_data) - - unmodified_pants_binary_create = run_target('binary') - self.assert_success(unmodified_pants_binary_create) - binary_run_output = invoke_pex_for_output(output_pex).decode('utf-8') - self.assertIn(initial_result_message, binary_run_output) - - # Modify one of the source files for this target so that the output is different. - cpp_source_file = os.path.join(self._binary_target_dir, 'some_more_math.cpp') - with open(cpp_source_file, 'r') as f: - orig_contents = f.read() - modified_contents = re.sub(r'3', '4', orig_contents) - with open(cpp_source_file, 'w') as f: - f.write(modified_contents) - - modified_result_message = 'x=3, f(x)=28' - - modified_pants_run = run_target('run') - self.assert_success(modified_pants_run) - self.assertIn(modified_result_message, modified_pants_run.stdout_data) - - modified_pants_binary_create = run_target('binary') - self.assert_success(modified_pants_binary_create) - binary_run_output = invoke_pex_for_output(output_pex).decode('utf-8') - self.assertIn(modified_result_message, binary_run_output) - def test_ctypes_native_language_interop(self): for variant in ToolchainVariant.allowed_values: self._assert_ctypes_interop_with_mock_buildroot(variant) diff --git a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py index fc0014720a0..1e92cdf6aaa 100644 --- a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py +++ b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py @@ -4,14 +4,15 @@ from __future__ import absolute_import, division, print_function, unicode_literals -from builtins import str -from textwrap import dedent +import re +import pex.resolver from twitter.common.collections import OrderedDict +from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.targets.python_distribution import PythonDistribution -from pants_test.backend.python.tasks.python_task_test_base import (check_wheel_platform_matches_host, - name_and_platform) +from pants.backend.python.targets.python_library import PythonLibrary +from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants_test.backend.python.tasks.util.build_local_dists_test_base import \ BuildLocalPythonDistributionsTestBase @@ -23,99 +24,104 @@ class TestBuildLocalDistsNativeSources(BuildLocalPythonDistributionsTestBase): ('src/python/dist:universal_dist', { 'key': 'universal', 'target_type': PythonDistribution, - 'sources': ['foo.py', 'bar.py', '__init__.py', 'setup.py'], + 'sources': ['__init__.py', 'setup.py'], 'filemap': { - 'src/python/dist/__init__.py': '', - 'src/python/dist/foo.py': 'print("foo")', - 'src/python/dist/bar.py': 'print("bar")', - 'src/python/dist/setup.py': dedent(""" - from setuptools import setup, find_packages - setup( - name='universal_dist', - version='0.0.0', - packages=find_packages() - ) - """) - } + '__init__.py': '', + 'setup.py': """\ +from setuptools import find_packages, setup +setup( + name='universal_dist', + version='0.0.0', + packages=find_packages() +) + """, + }, }), - ('src/python/plat_specific_dist:plat_specific_dist', { - 'key': 'platform_specific', + ('3rdparty/python:pycountry', { + 'key': 'pycountry', + 'target_type': PythonRequirementLibrary, + 'requirements': [ + PythonRequirement('pycountry==18.5.20'), + ], + }), + + ('src/python/setup_requires:setup_requires', { + 'key': 'setup_requires', 'target_type': PythonDistribution, - 'sources': ['__init__.py', 'setup.py', 'native_source.c'], + 'setup_requires': [ + '3rdparty/python:pycountry', + ], + 'sources': ['__init__.py', 'setup.py'], 'filemap': { - 'src/python/plat_specific_dist/__init__.py': '', - 'src/python/plat_specific_dist/setup.py': dedent(""" - from distutils.core import Extension - from setuptools import setup, find_packages - setup( - name='platform_specific_dist', - version='0.0.0', - packages=find_packages(), - extensions=[Extension('native_source', sources=['native_source.c'])] - ) - """), - 'src/python/plat_specific_dist/native_source.c': dedent(""" - #include - - static PyObject * native_source(PyObject *self, PyObject *args) { - return Py_BuildValue("s", "Hello from C!"); - } - - static PyMethodDef Methods[] = { - {"native_source", native_source, METH_VARARGS, ""}, - {NULL, NULL, 0, NULL} - }; - - #if PY_MAJOR_VERSION >= 3 - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "native_source", /* m_name */ - NULL, /* m_doc */ - -1, /* m_size */ - Methods, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ - }; - #endif - - #if PY_MAJOR_VERSION >= 3 - PyMODINIT_FUNC PyInit_native_source(void) { - return PyModule_Create(&moduledef); - } - #else - PyMODINIT_FUNC initnative_source(void) { - (void) Py_InitModule("native_source", Methods); - } - #endif - """), - } + '__init__.py': '', + 'setup.py': """\ +from setuptools import find_packages, setup +import pycountry + +us_country_string = pycountry.countries.get(alpha_2='US').name.replace(' ', '_').lower() + +setup( + name='setup_requires_dist_{}'.format(us_country_string), + version='0.0.0', + packages=find_packages(), +) + """, + }, }), + ('src/python/install_requires:install_requires', { + 'key': 'install_requires', + 'target_type': PythonDistribution, + 'sources': ['__init__.py', 'setup.py'], + 'filemap': { + '__init__.py': '', + 'setup.py': """\ +from setuptools import setup + +setup( + name='install_requires_dist', + version='0.0.0', + install_requires=['pycountry==17.1.2'], +) + """, + }, + }), + + ('src/python/install_requires:conflict', { + 'key': 'install_requires_conflict', + 'target_type': PythonLibrary, + 'dependencies': [ + '3rdparty/python:pycountry', + 'src/python/install_requires:install_requires', + ], + }), ]) - def test_python_create_universal_distribution(self): + def test_create_distribution(self): universal_dist = self.target_dict['universal'] - context, synthetic_target, snapshot_version = self._create_distribution_synthetic_target( - universal_dist) - self.assertEqual(['universal_dist==0.0.0+{}'.format(snapshot_version)], - [str(x.requirement) for x in synthetic_target.requirements.value]) - - local_wheel_products = context.products.get('local_wheels') - local_wheel = self._retrieve_single_product_at_target_base(local_wheel_products, universal_dist) - _, _, wheel_platform = name_and_platform(local_wheel) - self.assertEqual('any', wheel_platform) - - def test_python_create_platform_specific_distribution(self): - platform_specific_dist = self.target_dict['platform_specific'] - context, synthetic_target, snapshot_version = self._create_distribution_synthetic_target( - platform_specific_dist) - self.assertEqual(['platform_specific_dist==0.0.0+{}'.format(snapshot_version)], - [str(x.requirement) for x in synthetic_target.requirements.value]) - - local_wheel_products = context.products.get('local_wheels') - local_wheel = self._retrieve_single_product_at_target_base( - local_wheel_products, platform_specific_dist) - self.assertTrue(check_wheel_platform_matches_host(local_wheel)) + self._assert_dist_and_wheel_identity('universal_dist', '0.0.0', 'any', universal_dist) + + def test_python_dist_setup_requires(self): + setup_requires_dist = self.target_dict['setup_requires'] + self._assert_dist_and_wheel_identity( + 'setup_requires_dist_united_states', '0.0.0', 'any', + setup_requires_dist, extra_targets=[self.target_dict['pycountry']]) + + def test_install_requires(self): + install_requires_dist = self.target_dict['install_requires'] + self._assert_dist_and_wheel_identity( + 'install_requires_dist', '0.0.0', 'any', + install_requires_dist) + + def test_install_requires_conflict(self): + install_requires_dist = self.target_dict['install_requires'] + pycountry_req_lib = self.target_dict['pycountry'] + conflicting_lib = self.target_dict['install_requires_conflict'] + + with self.assertRaisesRegexp( + pex.resolver.Unsatisfiable, + re.escape('Could not satisfy all requirements for pycountry==18.5.20:')): + self._create_distribution_synthetic_target( + install_requires_dist, + extra_targets=[pycountry_req_lib, conflicting_lib]) diff --git a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions_integration.py b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions_integration.py new file mode 100644 index 00000000000..9b69570f7c8 --- /dev/null +++ b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions_integration.py @@ -0,0 +1,102 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import glob +import os +import re +from builtins import open + +from pants.util.collections import assert_single_element +from pants.util.contextutil import temporary_dir +from pants.util.process_handler import subprocess +from pants_test.pants_run_integration_test import PantsRunIntegrationTest +from pants_test.testutils.py2_compat import assertRegex + + +class BuildLocalPythonDistributionsIntegrationTest(PantsRunIntegrationTest): + hello_install_requires_dir = 'testprojects/src/python/python_distribution/hello_with_install_requires' + hello_setup_requires = 'examples/src/python/example/python_distribution/hello/setup_requires' + py_dist_test = 'testprojects/tests/python/example_test/python_distribution' + + def _assert_nation_and_greeting(self, output, punctuation='!'): + self.assertEquals("""\ +hello{} +United States +""".format(punctuation), output) + + def test_pydist_binary(self): + with temporary_dir() as tmp_dir: + pex = os.path.join(tmp_dir, 'main_with_no_conflict.pex') + command = [ + '--pants-distdir={}'.format(tmp_dir), + 'binary', + '{}:main_with_no_conflict'.format(self.hello_install_requires_dir), + ] + pants_run = self.run_pants(command=command) + self.assert_success(pants_run) + # Check that the pex was built. + self.assertTrue(os.path.isfile(pex)) + # Check that the pex runs. + output = subprocess.check_output(pex).decode('utf-8') + self._assert_nation_and_greeting(output) + # Check that we have exactly one wheel output. + single_wheel_output = assert_single_element(glob.glob(os.path.join(tmp_dir, '*.whl'))) + assertRegex(self, os.path.basename(single_wheel_output), + r'\A{}'.format(re.escape('hello_with_install_requires-1.0.0+'))) + + def test_pydist_run(self): + with temporary_dir() as tmp_dir: + command=[ + '--pants-distdir={}'.format(tmp_dir), + '--quiet', + 'run', + '{}:main_with_no_conflict'.format(self.hello_install_requires_dir)] + pants_run = self.run_pants(command=command) + self.assert_success(pants_run) + # Check that text was properly printed to stdout. + self._assert_nation_and_greeting(pants_run.stdout_data) + + def test_pydist_invalidation(self): + """Test that the current version of a python_dist() is resolved after modifying its sources.""" + hello_run = '{}:main_with_no_conflict'.format(self.hello_install_requires_dir) + + with self.mock_buildroot( + dirs_to_copy=[self.hello_install_requires_dir]) as buildroot, buildroot.pushd(): + run_target = lambda: self.run_pants_with_workdir( + command=['--quiet', 'run', hello_run], + workdir=os.path.join(buildroot.new_buildroot, '.pants.d'), + build_root=buildroot.new_buildroot, + ) + + unmodified_pants_run = run_target() + self.assert_success(unmodified_pants_run) + self._assert_nation_and_greeting(unmodified_pants_run.stdout_data) + + # Modify one of the source files for this target so that the output is different. + py_source_file = os.path.join( + self.hello_install_requires_dir, 'hello_package/hello.py') + with open(py_source_file, 'r') as f: + orig_contents = f.read() + # Replace hello! with hello? + modified_contents = re.sub('!', '?', orig_contents) + with open(py_source_file, 'w') as f: + f.write(modified_contents) + + modified_pants_run = run_target() + self.assert_success(modified_pants_run) + self._assert_nation_and_greeting(modified_pants_run.stdout_data, punctuation='?') + + def test_pydist_test(self): + with temporary_dir() as tmp_dir: + command=[ + '--pants-distdir={}'.format(tmp_dir), + 'test', + self.py_dist_test, + ] + pants_run = self.run_pants(command=command) + self.assert_success(pants_run) + # Make sure that there is no wheel output when 'binary' goal is not invoked. + self.assertEqual(0, len(glob.glob(os.path.join(tmp_dir, '*.whl')))) diff --git a/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py b/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py deleted file mode 100644 index 54ec8ea15f1..00000000000 --- a/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py +++ /dev/null @@ -1,214 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import glob -import os -import re -from builtins import open - -from pants.util.collections import assert_single_element -from pants.util.contextutil import environment_as, temporary_dir -from pants.util.dirutil import is_executable -from pants.util.process_handler import subprocess -from pants_test.pants_run_integration_test import PantsRunIntegrationTest -from pants_test.testutils.py2_compat import assertRegex - - -class PythonDistributionIntegrationTest(PantsRunIntegrationTest): - # The paths to both a project containing a simple C extension (to be packaged into a - # whl by setup.py) and an associated test to be consumed by the pants goals tested below. - fasthello_project = 'examples/src/python/example/python_distribution/hello/fasthello' - fasthello_tests = 'examples/tests/python/example/python_distribution/hello/test_fasthello' - fasthello_install_requires_dir = 'testprojects/src/python/python_distribution/fasthello_with_install_requires' - hello_setup_requires = 'examples/src/python/example/python_distribution/hello/setup_requires' - - def _assert_native_greeting(self, output): - self.assertIn('Hello from C!', output) - self.assertIn('Hello from C++!', output) - - def test_pants_binary(self): - with temporary_dir() as tmp_dir: - pex = os.path.join(tmp_dir, 'main.pex') - command=[ - '--pants-distdir={}'.format(tmp_dir), 'binary', '{}:main'.format(self.fasthello_project)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - # Check that the pex was built. - self.assertTrue(os.path.isfile(pex)) - # Check that the pex runs. - output = subprocess.check_output(pex).decode('utf-8') - self._assert_native_greeting(output) - # Check that we have exactly one wheel output. - single_wheel_output = assert_single_element(glob.glob(os.path.join(tmp_dir, '*.whl'))) - assertRegex(self, os.path.basename(single_wheel_output), - r'\A{}'.format(re.escape('fasthello-1.0.0+'))) - - def test_pants_run(self): - with temporary_dir() as tmp_dir: - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'run', - '{}:main'.format(self.fasthello_project)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - # Check that text was properly printed to stdout. - self._assert_native_greeting(pants_run.stdout_data) - - def test_invalidation(self): - """Test that the current version of a python_dist() is resolved after modifying its sources.""" - fasthello_run = '{}:main_with_no_conflict'.format(self.fasthello_install_requires_dir) - - with self.mock_buildroot( - dirs_to_copy=[self.fasthello_install_requires_dir]) as buildroot, buildroot.pushd(): - run_target = lambda: self.run_pants_with_workdir( - command=['run', fasthello_run], - workdir=os.path.join(buildroot.new_buildroot, '.pants.d'), - build_root=buildroot.new_buildroot, - ) - - unmodified_pants_run = run_target() - self.assert_success(unmodified_pants_run) - self.assertIn('Hello from C!\n', unmodified_pants_run.stdout_data) - - # Modify one of the source files for this target so that the output is different. - c_source_file = os.path.join(self.fasthello_install_requires_dir, 'c_greet.c') - with open(c_source_file, 'r') as f: - orig_contents = f.read() - modified_contents = re.sub('"Hello from C!"', '"Hello from C?"', orig_contents) - with open(c_source_file, 'w') as f: - f.write(modified_contents) - - modified_pants_run = run_target() - self.assert_success(modified_pants_run) - self.assertIn('Hello from C?\n', modified_pants_run.stdout_data) - - def test_pants_test(self): - with temporary_dir() as tmp_dir: - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'test', - '{}:fasthello'.format(self.fasthello_tests)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - # Make sure that there is no wheel output when 'binary' goal is not invoked. - self.assertEqual(len(glob.glob(os.path.join(tmp_dir, '*.whl'))), 0) - - def test_with_install_requires(self): - with temporary_dir() as tmp_dir: - pex = os.path.join(tmp_dir, 'main_with_no_conflict.pex') - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'run', - '{}:main_with_no_conflict'.format(self.fasthello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - self.assertIn('United States', pants_run.stdout_data) - command=['binary', '{}:main_with_no_conflict'.format(self.fasthello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - output = subprocess.check_output(pex).decode('utf-8') - self.assertIn('United States', output) - - def test_with_conflicting_transitive_deps(self): - command=['run', '{}:main_with_conflicting_dep'.format(self.fasthello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_failure(pants_run) - self.assertIn('pycountry', pants_run.stderr_data) - self.assertIn('fasthello', pants_run.stderr_data) - command=['binary', '{}:main_with_conflicting_dep'.format(self.fasthello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_failure(pants_run) - self.assertIn('pycountry', pants_run.stderr_data) - self.assertIn('fasthello', pants_run.stderr_data) - - def test_binary_dep_isolation_with_multiple_targets(self): - with temporary_dir() as tmp_dir: - pex1 = os.path.join(tmp_dir, 'main_with_no_conflict.pex') - pex2 = os.path.join(tmp_dir, 'main_with_no_pycountry.pex') - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'binary', - '{}:main_with_no_conflict'.format(self.fasthello_install_requires_dir), - '{}:main_with_no_pycountry'.format(self.fasthello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - # Check that the pex was built. - self.assertTrue(os.path.isfile(pex1)) - self.assertTrue(os.path.isfile(pex2)) - # Check that the pex 1 runs. - output = subprocess.check_output(pex1).decode('utf-8') - self._assert_native_greeting(output) - # Check that the pex 2 fails due to no python_dists leaked into it. - try: - subprocess.check_output(pex2) - except subprocess.CalledProcessError as e: - self.assertNotEqual(0, e.returncode) - - def test_pants_resolves_local_dists_for_current_platform_only(self): - # Test that pants will override pants.ini platforms config when building - # or running a target that depends on native (c or cpp) sources. - with temporary_dir() as tmp_dir: - pex = os.path.join(tmp_dir, 'main.pex') - pants_ini_config = { - 'python-setup': { - # If no targets exist declaring the nonexistent platform, this should be reduced to just - # ['current'] in PythonNativeCode#check_build_for_current_platform_only(). - 'platforms': ['current', 'this-platform-does_not-exist'], - }, - } - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'run', - '{}:main'.format(self.fasthello_project)] - pants_run = self.run_pants(command=command, config=pants_ini_config) - self.assert_success(pants_run) - - command=['binary', '{}:main'.format(self.fasthello_project)] - pants_run = self.run_pants(command=command, config=pants_ini_config) - self.assert_success(pants_run) - # Check that the pex was built. - self.assertTrue(os.path.isfile(pex)) - # Check that the pex runs. - output = subprocess.check_output(pex).decode('utf-8') - self._assert_native_greeting(output) - - def test_pants_tests_local_dists_for_current_platform_only(self): - with temporary_dir() as tmp_dir: - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'test', - '{}:fasthello'.format(self.fasthello_tests)] - pants_run = self.run_pants(command=command, config={ - 'python-setup': { - 'platforms': ['current'], - }, - }) - self.assert_success(pants_run) - - def test_python_distribution_with_setup_requires(self): - # Validate that setup_requires dependencies are present and functional. - # PANTS_TEST_SETUP_REQUIRES triggers test functionality in this particular setup.py. - with environment_as(PANTS_TEST_SETUP_REQUIRES='1'): - command=['run', '{}:main'.format(self.hello_setup_requires)] - pants_run = self.run_pants(command=command) - - command=['run', '{}:main'.format(self.hello_setup_requires)] - pants_run = self.run_pants(command=command) - - with temporary_dir() as tmp_dir: - pex = os.path.join(tmp_dir, 'main.pex') - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'binary', - '{}:main'.format(self.hello_setup_requires), - ] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - # Check that the pex was built. - self.assertTrue(is_executable(pex)) - # Check that the pex runs. - output = subprocess.check_output(pex).decode('utf-8') - self.assertEqual('Hello, world!\n', output) diff --git a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py index 3831dbed46d..09199db2442 100644 --- a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py +++ b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py @@ -4,18 +4,23 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import os import re -from builtins import next +from builtins import next, str from pants.backend.native.register import rules as native_backend_rules from pants.backend.native.subsystems.libc_dev import LibcDev from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.tasks.build_local_python_distributions import \ BuildLocalPythonDistributions +from pants.backend.python.tasks.resolve_requirements import ResolveRequirements from pants.backend.python.tasks.select_interpreter import SelectInterpreter +from pants.build_graph.address import Address from pants.util.collections import assert_single_element -from pants.util.memo import memoized_property -from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase +from pants.util.memo import memoized_method +from pants.util.meta import classproperty +from pants_test.backend.python.tasks.python_task_test_base import (PythonTaskTestBase, + name_and_platform) from pants_test.engine.scheduler_test_base import SchedulerTestBase @@ -25,22 +30,37 @@ class BuildLocalPythonDistributionsTestBase(PythonTaskTestBase, SchedulerTestBas def task_type(cls): return BuildLocalPythonDistributions - # This is an informally-specified nested dict -- see ../test_ctypes.py for an example. Special - # keys are 'key' (used to index into `self.target_dict`) and 'filemap' (creates files at the - # specified relative paths). The rest of the keys are fed into `self.make_target()`. An - # `OrderedDict` of 2-tuples may be used if targets need to be created in a specific order (e.g. if - # they have dependencies on each other). - _dist_specs = None - # By default, we just use a `BuildLocalPythonDistributions` task. When testing with C/C++ targets, - # we want to compile and link them as well to get the resulting dist to build, so we add those - # task types here and execute them beforehand. - _extra_relevant_task_types = [SelectInterpreter] - - @memoized_property - def _all_other_synthesized_task_types(self): + @classproperty + def _dist_specs(cls): + """ + This is an informally-specified nested dict -- see ../test_ctypes.py for an example. Special + keys are 'key' (used to index into `self.target_dict`) and 'filemap' (creates files at the + specified relative paths). The rest of the keys are fed into `self.make_target()`. An + `OrderedDict` of 2-tuples may be used if targets need to be created in a specific order (e.g. if + they have dependencies on each other). + """ + raise NotImplementedError('_dist_specs must be implemented!') + + @classproperty + def _run_before_task_types(cls): + """ + By default, we just use a `BuildLocalPythonDistributions` task. When testing with C/C++ targets, + we want to compile and link them as well to get the resulting dist to build, so we add those + task types here and execute them beforehand. + """ + return [SelectInterpreter] + + @classproperty + def _run_after_task_types(cls): + """Tasks to run after local dists are built, similar to `_run_before_task_types`.""" + return [ResolveRequirements] + + @memoized_method + def _synthesize_task_types(self, task_types=()): return [ self.synthesize_task_subtype(tsk, '__tmp_{}'.format(tsk.__name__)) - for tsk in self._extra_relevant_task_types + # TODO: make @memoized_method convert lists to tuples for hashing! + for tsk in task_types ] def setUp(self): @@ -48,20 +68,31 @@ def setUp(self): self.target_dict = {} - # Create a python_dist() target from each specification and insert it into `self.target_dict`. - for target_spec, file_spec in self._dist_specs.items(): - file_spec = file_spec.copy() - filemap = file_spec.pop('filemap') + # Create a target from each specification and insert it into `self.target_dict`. + for target_spec, target_kwargs in self._dist_specs.items(): + unprocessed_kwargs = target_kwargs.copy() + + target_base = Address.parse(target_spec).spec_path + + # Populate the target's owned files from the specification. + filemap = unprocessed_kwargs.pop('filemap', {}) for rel_path, content in filemap.items(): - self.create_file(rel_path, content) + buildroot_path = os.path.join(target_base, rel_path) + self.create_file(buildroot_path, content) - key = file_spec.pop('key') + # Ensure any dependencies exist in the target dict (`_dist_specs` must then be an + # OrderedDict). + # The 'key' is used to access the target in `self.target_dict`. + key = unprocessed_kwargs.pop('key') dep_targets = [] - for dep_spec in file_spec.pop('dependencies', []): + for dep_spec in unprocessed_kwargs.pop('dependencies', []): existing_tgt_key = self._dist_specs[dep_spec]['key'] dep_targets.append(self.target_dict[existing_tgt_key]) - python_dist_tgt = self.make_target(spec=target_spec, dependencies=dep_targets, **file_spec) - self.target_dict[key] = python_dist_tgt + + # Register the generated target. + generated_target = self.make_target( + spec=target_spec, dependencies=dep_targets, **unprocessed_kwargs) + self.target_dict[key] = generated_target def _all_specified_targets(self): return list(self.target_dict.values()) @@ -105,15 +136,16 @@ def _create_task(self, task_type, context): return task_type(context, self.test_workdir) def _create_distribution_synthetic_target(self, python_dist_target, extra_targets=[]): - + run_before_synthesized_task_types = self._synthesize_task_types(tuple(self._run_before_task_types)) python_create_distributions_task_type = self._testing_task_type - all_synthesized_task_types = self._all_other_synthesized_task_types + [ + run_after_synthesized_task_types = self._synthesize_task_types(tuple(self._run_after_task_types)) + all_synthesized_task_types = run_before_synthesized_task_types + [ python_create_distributions_task_type, - ] + ] + run_after_synthesized_task_types context = self._scheduling_context( target_roots=([python_dist_target] + extra_targets), - for_task_types=(all_synthesized_task_types), + for_task_types=all_synthesized_task_types, for_subsystems=[PythonRepos, LibcDev], # TODO(#6848): we should be testing all of these with both of our toolchains. options={ @@ -123,19 +155,23 @@ def _create_distribution_synthetic_target(self, python_dist_target, extra_target }) self.assertEqual(set(self._all_specified_targets()), set(context.build_graph.targets())) - all_other_task_instances = [ + run_before_task_instances = [ self._create_task(task_type, context) - for task_type in self._all_other_synthesized_task_types + for task_type in run_before_synthesized_task_types ] python_create_distributions_task_instance = self._create_task( - python_create_distributions_task_type, - context) + python_create_distributions_task_type, context) + run_after_task_instances = [ + self._create_task(task_type, context) + for task_type in run_after_synthesized_task_types + ] + all_task_instances = run_before_task_instances + [ + python_create_distributions_task_instance + ] + run_after_task_instances - for tsk in all_other_task_instances: + for tsk in all_task_instances: tsk.execute() - python_create_distributions_task_instance.execute() - synthetic_tgts = set(context.build_graph.targets()) - set(self._all_specified_targets()) self.assertEqual(1, len(synthetic_tgts)) synthetic_target = next(iter(synthetic_tgts)) @@ -144,3 +180,20 @@ def _create_distribution_synthetic_target(self, python_dist_target, extra_target python_create_distributions_task_instance, python_dist_target) return context, synthetic_target, snapshot_version + + def _assert_dist_and_wheel_identity(self, expected_name, expected_version, expected_platform, + dist_target, **kwargs): + context, synthetic_target, fingerprint_suffix = self._create_distribution_synthetic_target( + dist_target, **kwargs) + resulting_dist_req = assert_single_element(synthetic_target.requirements.value) + expected_snapshot_version = '{}+{}'.format(expected_version, fingerprint_suffix) + self.assertEquals( + '{}=={}'.format(expected_name, expected_snapshot_version), + str(resulting_dist_req.requirement)) + + local_wheel_products = context.products.get('local_wheels') + local_wheel = self._retrieve_single_product_at_target_base(local_wheel_products, dist_target) + dist, version, platform = name_and_platform(local_wheel) + self.assertEquals(dist, expected_name) + self.assertEquals(version, expected_snapshot_version) + self.assertEquals(platform, expected_platform)