diff --git a/build-support/known_py3_pex_failures.txt b/build-support/known_py3_pex_failures.txt index c06989556201..92b3341ae2c3 100644 --- a/build-support/known_py3_pex_failures.txt +++ b/build-support/known_py3_pex_failures.txt @@ -3,8 +3,6 @@ tests/python/pants_test/backend/jvm/tasks:jvm_platform_analysis_integration tests/python/pants_test/backend/jvm/tasks:scala_repl_integration tests/python/pants_test/backend/project_info/tasks:idea_plugin_integration tests/python/pants_test/backend/python/tasks:integration -tests/python/pants_test/backend/python/tasks:isort_run_integration -tests/python/pants_test/backend/python/tasks:python_native_code_testing_1 tests/python/pants_test/backend/python:integration tests/python/pants_test/base:exiter_integration tests/python/pants_test/engine/legacy:changed_integration diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/BUILD b/examples/src/python/example/python_distribution/hello/setup_requires/BUILD deleted file mode 100644 index ca852ee8104b..000000000000 --- a/examples/src/python/example/python_distribution/hello/setup_requires/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -# Like Hello world, but built with a python_dist. -# python_dist allows you to use setup.py to depend on C/C++ extensions. - -python_dist( - name='hello', - sources=[ - 'hello_package/hello.py', - 'hello_package/__init__.py', - 'setup.cfg', - 'setup.py' - ], - setup_requires=[ - 'examples/src/python/example/python_distribution/hello/setup_requires:setup_requires_lib' - ] -) - -python_requirement_library( - name = "setup_requires_lib", - requirements = [ - python_requirement("pycountry==18.5.20"), - ], -) - -python_binary( - name='main', - source='main.py', - dependencies=[ - ':hello', - ], - platforms=['current'] -) diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/__init__.py b/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/hello.py b/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/hello.py deleted file mode 100644 index d24910327fcf..000000000000 --- a/examples/src/python/example/python_distribution/hello/setup_requires/hello_package/hello.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - - -def hello(): - return 'Hello, world!' diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/main.py b/examples/src/python/example/python_distribution/hello/setup_requires/main.py deleted file mode 100644 index cf04ba357b70..000000000000 --- a/examples/src/python/example/python_distribution/hello/setup_requires/main.py +++ /dev/null @@ -1,11 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -from hello_package import hello - - -if __name__ == '__main__': - print(hello.hello()) diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg b/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg deleted file mode 100644 index 3480374bc2f2..000000000000 --- a/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 \ No newline at end of file diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/setup.py b/examples/src/python/example/python_distribution/hello/setup_requires/setup.py deleted file mode 100644 index 3a035a939573..000000000000 --- a/examples/src/python/example/python_distribution/hello/setup_requires/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import os -from setuptools import setup, find_packages - -# We require pycountry with setup_requires argument to this setup script's -# corresponding python_dist. -import pycountry - -# This is for testing purposes so we can assert that setup_requires is functioning -# correctly (because Pants swallows print statements). -if os.getenv('PANTS_TEST_SETUP_REQUIRES', ''): - output = [bytes(pycountry)] - output.extend(os.listdir(os.getenv('PYTHONPATH', ''))) - raise Exception(str(output)) - -setup( - name='hello', - version='1.0.0', - packages=find_packages(), -) diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index 0edfd0952518..1b7dc702905f 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -7,8 +7,6 @@ from builtins import str from collections import defaultdict -from pex.pex import PEX - from pants.backend.native.subsystems.native_toolchain import NativeToolchain from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.python.python_requirement import PythonRequirement @@ -19,7 +17,7 @@ from pants.binaries.executable_pex_tool import ExecutablePexTool from pants.subsystem.subsystem import Subsystem from pants.util.memo import memoized_property -from pants.util.objects import SubclassesOf, datatype +from pants.util.objects import SubclassesOf class PythonNativeCode(Subsystem): @@ -151,9 +149,3 @@ def base_requirements(self): PythonRequirement('setuptools=={}'.format(self.python_setup.setuptools_version)), PythonRequirement('wheel=={}'.format(self.python_setup.wheel_version)), ] - - -class SetupPyExecutionEnvironment(datatype([ - # If None, don't set PYTHONPATH in the setup.py environment. - ('setup_requires_pex', PEX), -])): pass diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index 12763a4fd56e..93bf53b5e6f9 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -17,8 +17,7 @@ from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.subsystems.pex_build_util import is_local_python_dist from pants.backend.python.subsystems.python_native_code import (BuildSetupRequiresPex, - PythonNativeCode, - SetupPyExecutionEnvironment) + PythonNativeCode) from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.base.build_environment import get_buildroot from pants.base.exceptions import TargetDefinitionException, TaskError @@ -32,6 +31,7 @@ from pants.util.strutil import safe_shlex_join +# TODO: make this a SimpleCodegenTask!!! class BuildLocalPythonDistributions(Task): """Create python distributions (.whl) from python_dist targets.""" @@ -212,18 +212,15 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t setup_reqs_pex_path = os.path.join( setup_requires_dir, 'setup-requires-{}.pex'.format(versioned_target_fingerprint)) - extra_reqs = list(setup_reqs_to_resolve or []) setup_requires_pex = self._build_setup_requires_pex_settings.bootstrap( - interpreter, setup_reqs_pex_path, extra_reqs=extra_reqs) + interpreter, setup_reqs_pex_path, extra_reqs=setup_reqs_to_resolve) self.context.log.debug('Using pex file as setup.py interpreter: {}' .format(setup_requires_pex)) - setup_py_execution_environment = SetupPyExecutionEnvironment(setup_requires_pex) - self._create_dist( dist_target, dist_output_dir, - setup_py_execution_environment, + setup_requires_pex, versioned_target_fingerprint, is_platform_specific) @@ -256,7 +253,7 @@ def _generate_snapshot_bdist_wheel_argv(self, snapshot_fingerprint, is_platform_ def _create_dist(self, dist_tgt, dist_target_dir, - setup_py_execution_environment, + setup_requires_pex, snapshot_fingerprint, is_platform_specific): """Create a .whl file for the specified python_distribution target.""" @@ -265,8 +262,6 @@ def _create_dist(self, setup_py_snapshot_version_argv = self._generate_snapshot_bdist_wheel_argv( snapshot_fingerprint, is_platform_specific) - setup_requires_pex = setup_py_execution_environment.setup_requires_pex - cmd = safe_shlex_join(setup_requires_pex.cmdline(setup_py_snapshot_version_argv)) with self.context.new_workunit('setup.py', cmd=cmd, labels=[WorkUnitLabel.TOOL]) as workunit: with pushd(dist_target_dir): @@ -277,7 +272,7 @@ def _create_dist(self, raise self.BuildLocalPythonDistributionsError( "Installation of python distribution from target {target} into directory {into_dir} " "failed (return value of run() was: {rc!r}).\n" - "The chosen interpreter was: {interpreter}.\n" + "The pex with any requirements is located at: {interpreter}.\n" "The host system's compiler and linker were used.\n" "The setup command was: {command}." .format(target=dist_tgt, @@ -307,7 +302,9 @@ def _get_whl_from_dir(cls, install_dir): dists = glob.glob(os.path.join(dist_dir, '*.whl')) if len(dists) == 0: raise cls.BuildLocalPythonDistributionsError( - 'No distributions were produced by python_create_distribution task.') + 'No distributions were produced by python_create_distribution task.\n' + 'dist_dir: {}, install_dir: {}' + .format(dist_dir, install_dir)) if len(dists) > 1: # TODO: is this ever going to happen? raise cls.BuildLocalPythonDistributionsError('Ambiguous local python distributions found: {}' diff --git a/src/python/pants/backend/python/tasks/resolve_requirements.py b/src/python/pants/backend/python/tasks/resolve_requirements.py index eadde759a90d..ea8b74fb9bd7 100644 --- a/src/python/pants/backend/python/tasks/resolve_requirements.py +++ b/src/python/pants/backend/python/tasks/resolve_requirements.py @@ -14,6 +14,8 @@ class ResolveRequirements(ResolveRequirementsTaskBase): """Resolve external Python requirements.""" REQUIREMENTS_PEX = 'python_requirements_pex' + options_scope = 'resolve-requirements' + @classmethod def product_types(cls): return [cls.REQUIREMENTS_PEX] diff --git a/testprojects/src/python/python_distribution/hello_with_install_requires/BUILD b/testprojects/src/python/python_distribution/hello_with_install_requires/BUILD index 21f2c3ed9972..93f9b181dd13 100644 --- a/testprojects/src/python/python_distribution/hello_with_install_requires/BUILD +++ b/testprojects/src/python/python_distribution/hello_with_install_requires/BUILD @@ -33,11 +33,3 @@ python_requirement_library( python_requirement('pycountry==17.9.23'), ] ) - -python_binary( - name='main_with_no_pycountry', - source='main.py', - dependencies=[ - ':hello_with_install_requires' - ], -) diff --git a/examples/tests/python/example_test/python_distribution/BUILD b/testprojects/tests/python/example_test/python_distribution/BUILD similarity index 100% rename from examples/tests/python/example_test/python_distribution/BUILD rename to testprojects/tests/python/example_test/python_distribution/BUILD diff --git a/examples/tests/python/example_test/python_distribution/test_hello.py b/testprojects/tests/python/example_test/python_distribution/test_hello.py similarity index 78% rename from examples/tests/python/example_test/python_distribution/test_hello.py rename to testprojects/tests/python/example_test/python_distribution/test_hello.py index b2e156f5c061..ce206f9b69d5 100644 --- a/examples/tests/python/example_test/python_distribution/test_hello.py +++ b/testprojects/tests/python/example_test/python_distribution/test_hello.py @@ -6,10 +6,10 @@ import unittest -from hello import hello_string +from hello_package import hello class HelloTest(unittest.TestCase): def test_hello_import(self): - self.assertEqual('hello!', hello_string()) + self.assertEqual('hello!', hello.hello_string()) diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index cce50d37806a..072b3380e38f 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -22,85 +22,8 @@ python_library( ] ) -python_native_code_test_files = [ - 'test_build_local_python_distributions.py', - 'test_build_local_python_distributions_integration.py', - 'test_ctypes.py', - 'test_ctypes_integration.py', -] - -python_tests( - name='python_native_code_testing_3', - sources=[ - python_native_code_test_files[0], - ], - dependencies=[ - '3rdparty/python:future', - '3rdparty/python/twitter/commons:twitter.common.collections', - 'src/python/pants/backend/native', - 'src/python/pants/backend/native/targets', - 'src/python/pants/backend/python:plugin', - 'src/python/pants/backend/python/targets', - 'src/python/pants/backend/python/tasks', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:int-test', - 'tests/python/pants_test/backend/python/tasks/util', - 'tests/python/pants_test/engine:scheduler_test_base', - 'tests/python/pants_test/testutils:py2_compat', - ], - tags={'platform_specific_behavior', 'integration'}, - timeout=2400, -) - -python_tests( - name='python_native_code_testing_2', - sources=[ - python_native_code_test_files[1], - ], - dependencies=[ - '3rdparty/python:future', - '3rdparty/python/twitter/commons:twitter.common.collections', - 'src/python/pants/backend/native', - 'src/python/pants/backend/native/targets', - 'src/python/pants/backend/python:plugin', - 'src/python/pants/backend/python/targets', - 'src/python/pants/backend/python/tasks', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:int-test', - 'tests/python/pants_test/backend/python/tasks/util', - 'tests/python/pants_test/engine:scheduler_test_base', - 'tests/python/pants_test/testutils:py2_compat', - ], - tags={'platform_specific_behavior', 'integration'}, - timeout=2400, -) - python_tests( - name='python_native_code_testing_1', - sources=python_native_code_test_files[2:], - dependencies=[ - '3rdparty/python:future', - '3rdparty/python/twitter/commons:twitter.common.collections', - 'src/python/pants/backend/native', - 'src/python/pants/backend/native/targets', - 'src/python/pants/backend/python:plugin', - 'src/python/pants/backend/python/targets', - 'src/python/pants/backend/python/tasks', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:int-test', - 'tests/python/pants_test/backend/python/tasks/util', - 'tests/python/pants_test/engine:scheduler_test_base', - 'tests/python/pants_test/testutils:py2_compat', - ], - tags={'platform_specific_behavior', 'integration'}, - timeout=2400, -) - -python_tests( - sources=globs('test_*.py', exclude=([globs('*_integration.py')] + python_native_code_test_files)), + sources=globs('test_*.py', exclude=([globs('*_integration.py')])), dependencies=[ '3rdparty/python/twitter/commons:twitter.common.collections', '3rdparty/python/twitter/commons:twitter.common.dirutil', @@ -128,6 +51,7 @@ python_tests( 'src/python/pants/util:process_handler', 'src/python/pants/util:py2_compat', 'tests/python/pants_test/backend/python:interpreter_selection_utils', + 'tests/python/pants_test/backend/python/tasks/util', 'tests/python/pants_test/engine:scheduler_test_base', 'tests/python/pants_test/subsystem:subsystem_utils', 'tests/python/pants_test/tasks:task_test_base', @@ -139,7 +63,7 @@ python_tests( python_tests( name='integration', - sources=globs('*_integration.py', exclude=python_native_code_test_files), + sources=globs('*_integration.py'), dependencies=[ '3rdparty/python:pex', 'src/python/pants/base:build_environment', @@ -154,28 +78,3 @@ python_tests( tags={'integration'}, timeout=2400 ) - -python_tests( - name='isort_run_integration', - sources=['test_isort_run_integration.py'], - dependencies=[ - 'tests/python/pants_test:int-test', - 'tests/python/pants_test/backend/python/tasks:python_task_test_base', - ], - tags={'integration'}, -) - -python_tests( - name='isort_run', - sources=['test_isort_run.py'], - dependencies=[ - '3rdparty/python:future', - 'src/python/pants/util:dirutil', - 'tests/python/pants_test/backend/python/tasks:python_task_test_base', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:test_base', - 'tests/python/pants_test/subsystem:subsystem_utils', - 'tests/python/pants_test:task_test_base', - ], - timeout=300 -) diff --git a/tests/python/pants_test/backend/python/tasks/native/BUILD b/tests/python/pants_test/backend/python/tasks/native/BUILD new file mode 100644 index 000000000000..7bbccd25921a --- /dev/null +++ b/tests/python/pants_test/backend/python/tasks/native/BUILD @@ -0,0 +1,63 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_tests( + sources=globs('test_*.py', exclude=[globs('*_integration.py')]), + dependencies=[ + '3rdparty/python/twitter/commons:twitter.common.collections', + '3rdparty/python/twitter/commons:twitter.common.dirutil', + '3rdparty/python:coverage', + '3rdparty/python:future', + '3rdparty/python:mock', + '3rdparty/python:pex', + 'src/python/pants/backend/native', + 'src/python/pants/backend/python/subsystems', + 'src/python/pants/backend/python/targets', + 'src/python/pants/backend/python/tasks', + 'src/python/pants/backend/python:interpreter_cache', + 'src/python/pants/backend/python:plugin', + 'src/python/pants/backend/python:python_artifact', + 'src/python/pants/backend/python:python_requirement', + 'src/python/pants/base:build_root', + 'src/python/pants/base:exceptions', + 'src/python/pants/base:run_info', + 'src/python/pants/build_graph', + 'src/python/pants/fs', + 'src/python/pants/source', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', + 'src/python/pants/util:py2_compat', + 'tests/python/pants_test/backend/python:interpreter_selection_utils', + 'tests/python/pants_test/backend/python/tasks:python_task_test_base', + 'tests/python/pants_test/backend/python/tasks/util', + 'tests/python/pants_test/engine:scheduler_test_base', + 'tests/python/pants_test/subsystem:subsystem_utils', + 'tests/python/pants_test/tasks:task_test_base', + 'tests/python/pants_test:task_test_base', + ], + tags={'platform_specific_behavior'}, + timeout=600, +) + +python_tests( + name='integration', + sources=globs('*_integration.py'), + dependencies=[ + '3rdparty/python:future', + '3rdparty/python/twitter/commons:twitter.common.collections', + 'src/python/pants/backend/native', + 'src/python/pants/backend/native/targets', + 'src/python/pants/backend/python:plugin', + 'src/python/pants/backend/python/targets', + 'src/python/pants/backend/python/tasks', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', + 'tests/python/pants_test:int-test', + 'tests/python/pants_test/backend/python/tasks/util', + 'tests/python/pants_test/engine:scheduler_test_base', + 'tests/python/pants_test/testutils:py2_compat', + ], + tags={'platform_specific_behavior', 'integration'}, + timeout=2400, +) diff --git a/tests/python/pants_test/backend/python/tasks/test_ctypes.py b/tests/python/pants_test/backend/python/tasks/native/test_ctypes.py similarity index 73% rename from tests/python/pants_test/backend/python/tasks/test_ctypes.py rename to tests/python/pants_test/backend/python/tasks/native/test_ctypes.py index 24ad01886f83..d1b014a45ea0 100644 --- a/tests/python/pants_test/backend/python/tasks/test_ctypes.py +++ b/tests/python/pants_test/backend/python/tasks/native/test_ctypes.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals from builtins import str -from textwrap import dedent from twitter.common.collections import OrderedDict @@ -15,6 +14,7 @@ from pants.backend.native.tasks.cpp_compile import CppCompile from pants.backend.native.tasks.link_shared_libraries import LinkSharedLibraries from pants.backend.python.targets.python_distribution import PythonDistribution +from pants.util.meta import classproperty from pants_test.backend.python.tasks.python_task_test_base import check_wheel_platform_matches_host from pants_test.backend.python.tasks.util.build_local_dists_test_base import \ BuildLocalPythonDistributionsTestBase @@ -22,11 +22,13 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTestBase): - _extra_relevant_task_types = ([ - CCompile, - CppCompile, - LinkSharedLibraries, - ] + BuildLocalPythonDistributionsTestBase._extra_relevant_task_types) + @classproperty + def _run_before_task_types(cls): + return [ + CCompile, + CppCompile, + LinkSharedLibraries, + ] + super(TestBuildLocalDistsWithCtypesNativeSources, cls)._run_before_task_types _dist_specs = OrderedDict([ @@ -36,14 +38,14 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTe 'ctypes_native_library': NativeArtifact(lib_name='c-math-lib'), 'sources': ['c_math_lib.c', 'c_math_lib.h'], 'filemap': { - 'src/python/plat_specific_c_dist/c_math_lib.c': dedent(""" - #include "c_math_lib.h" - int add_two(int x) { return x + 2; } -"""), - 'src/python/plat_specific_c_dist/c_math_lib.h': dedent(""" - int add_two(int); -"""), - } + 'c_math_lib.c': """\ +#include "c_math_lib.h" +int add_two(int x) { return x + 2; } + """, + 'c_math_lib.h': """\ +int add_two(int); + """, + }, }), ('src/python/plat_specific_c_dist:plat_specific_ctypes_c_dist', { @@ -52,17 +54,17 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTe 'sources': ['__init__.py', 'setup.py'], 'dependencies': ['src/python/plat_specific_c_dist:ctypes_c_library'], 'filemap': { - 'src/python/plat_specific_c_dist/__init__.py': '', - 'src/python/plat_specific_c_dist/setup.py': dedent(""" - from setuptools import setup, find_packages - setup( - name='platform_specific_ctypes_c_dist', - version='0.0.0', - packages=find_packages(), - data_files=[('', ['libc-math-lib.so'])], - ) - """), - } + '__init__.py': '', + 'setup.py': """\ +from setuptools import setup, find_packages +setup( + name='platform_specific_ctypes_c_dist', + version='0.0.0', + packages=find_packages(), + data_files=[('', ['libc-math-lib.so'])], +) + """, + }, }), ('src/python/plat_specific_cpp_dist:ctypes_cpp_library', { @@ -71,13 +73,13 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTe 'ctypes_native_library': NativeArtifact(lib_name='cpp-math-lib'), 'sources': ['cpp_math_lib.cpp', 'cpp_math_lib.hpp'], 'filemap': { - 'src/python/plat_specific_cpp_dist/cpp_math_lib.cpp': dedent(""" + 'cpp_math_lib.cpp': """\ #include "cpp_math_lib.hpp" int add_two(int x) { return (x++) + 1; } -"""), - 'src/python/plat_specific_cpp_dist/cpp_math_lib.hpp': dedent(""" + """, + 'cpp_math_lib.hpp': """\ int add_two(int); -"""), + """, }, }), @@ -87,17 +89,17 @@ class TestBuildLocalDistsWithCtypesNativeSources(BuildLocalPythonDistributionsTe 'sources': ['__init__.py', 'setup.py'], 'dependencies': ['src/python/plat_specific_cpp_dist:ctypes_cpp_library'], 'filemap': { - 'src/python/plat_specific_cpp_dist/__init__.py': '', - 'src/python/plat_specific_cpp_dist/setup.py': dedent(""" - from setuptools import setup, find_packages - setup( - name='platform_specific_ctypes_cpp_dist', - version='0.0.0', - packages=find_packages(), - data_files=[('', ['libcpp-math-lib.so'])], - ) - """), - } + '__init__.py': '', + 'setup.py': """\ +from setuptools import setup, find_packages +setup( + name='platform_specific_ctypes_cpp_dist', + version='0.0.0', + packages=find_packages(), + data_files=[('', ['libcpp-math-lib.so'])], +) + """, + }, }), ]) diff --git a/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py b/tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py similarity index 100% rename from tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py rename to tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py diff --git a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py index d7317cacddce..1e92cdf6aaaf 100644 --- a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py +++ b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py @@ -4,13 +4,15 @@ from __future__ import absolute_import, division, print_function, unicode_literals -from builtins import str -from textwrap import dedent +import re +import pex.resolver from twitter.common.collections import OrderedDict +from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.targets.python_distribution import PythonDistribution -from pants_test.backend.python.tasks.python_task_test_base import name_and_platform +from pants.backend.python.targets.python_library import PythonLibrary +from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants_test.backend.python.tasks.util.build_local_dists_test_base import \ BuildLocalPythonDistributionsTestBase @@ -22,32 +24,104 @@ class TestBuildLocalDistsNativeSources(BuildLocalPythonDistributionsTestBase): ('src/python/dist:universal_dist', { 'key': 'universal', 'target_type': PythonDistribution, - 'sources': ['foo.py', 'bar.py', '__init__.py', 'setup.py'], + 'sources': ['__init__.py', 'setup.py'], 'filemap': { - 'src/python/dist/__init__.py': '', - 'src/python/dist/foo.py': 'print("foo")', - 'src/python/dist/bar.py': 'print("bar")', - 'src/python/dist/setup.py': dedent(""" - from setuptools import setup, find_packages - setup( - name='universal_dist', - version='0.0.0', - packages=find_packages() - ) - """) - } + '__init__.py': '', + 'setup.py': """\ +from setuptools import find_packages, setup +setup( + name='universal_dist', + version='0.0.0', + packages=find_packages() +) + """, + }, }), + ('3rdparty/python:pycountry', { + 'key': 'pycountry', + 'target_type': PythonRequirementLibrary, + 'requirements': [ + PythonRequirement('pycountry==18.5.20'), + ], + }), + + ('src/python/setup_requires:setup_requires', { + 'key': 'setup_requires', + 'target_type': PythonDistribution, + 'setup_requires': [ + '3rdparty/python:pycountry', + ], + 'sources': ['__init__.py', 'setup.py'], + 'filemap': { + '__init__.py': '', + 'setup.py': """\ +from setuptools import find_packages, setup +import pycountry + +us_country_string = pycountry.countries.get(alpha_2='US').name.replace(' ', '_').lower() + +setup( + name='setup_requires_dist_{}'.format(us_country_string), + version='0.0.0', + packages=find_packages(), +) + """, + }, + }), + + ('src/python/install_requires:install_requires', { + 'key': 'install_requires', + 'target_type': PythonDistribution, + 'sources': ['__init__.py', 'setup.py'], + 'filemap': { + '__init__.py': '', + 'setup.py': """\ +from setuptools import setup + +setup( + name='install_requires_dist', + version='0.0.0', + install_requires=['pycountry==17.1.2'], +) + """, + }, + }), + + ('src/python/install_requires:conflict', { + 'key': 'install_requires_conflict', + 'target_type': PythonLibrary, + 'dependencies': [ + '3rdparty/python:pycountry', + 'src/python/install_requires:install_requires', + ], + }), ]) - def test_python_create_universal_distribution(self): + def test_create_distribution(self): universal_dist = self.target_dict['universal'] - context, synthetic_target, snapshot_version = self._create_distribution_synthetic_target( - universal_dist) - self.assertEqual(['universal_dist==0.0.0+{}'.format(snapshot_version)], - [str(x.requirement) for x in synthetic_target.requirements.value]) - - local_wheel_products = context.products.get('local_wheels') - local_wheel = self._retrieve_single_product_at_target_base(local_wheel_products, universal_dist) - _, _, wheel_platform = name_and_platform(local_wheel) - self.assertEqual('any', wheel_platform) + self._assert_dist_and_wheel_identity('universal_dist', '0.0.0', 'any', universal_dist) + + def test_python_dist_setup_requires(self): + setup_requires_dist = self.target_dict['setup_requires'] + self._assert_dist_and_wheel_identity( + 'setup_requires_dist_united_states', '0.0.0', 'any', + setup_requires_dist, extra_targets=[self.target_dict['pycountry']]) + + def test_install_requires(self): + install_requires_dist = self.target_dict['install_requires'] + self._assert_dist_and_wheel_identity( + 'install_requires_dist', '0.0.0', 'any', + install_requires_dist) + + def test_install_requires_conflict(self): + install_requires_dist = self.target_dict['install_requires'] + pycountry_req_lib = self.target_dict['pycountry'] + conflicting_lib = self.target_dict['install_requires_conflict'] + + with self.assertRaisesRegexp( + pex.resolver.Unsatisfiable, + re.escape('Could not satisfy all requirements for pycountry==18.5.20:')): + self._create_distribution_synthetic_target( + install_requires_dist, + extra_targets=[pycountry_req_lib, conflicting_lib]) diff --git a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions_integration.py b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions_integration.py index 133f1fd5468b..9b69570f7c89 100644 --- a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions_integration.py +++ b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions_integration.py @@ -10,8 +10,7 @@ from builtins import open from pants.util.collections import assert_single_element -from pants.util.contextutil import environment_as, temporary_dir -from pants.util.dirutil import is_executable +from pants.util.contextutil import temporary_dir from pants.util.process_handler import subprocess from pants_test.pants_run_integration_test import PantsRunIntegrationTest from pants_test.testutils.py2_compat import assertRegex @@ -20,16 +19,20 @@ class BuildLocalPythonDistributionsIntegrationTest(PantsRunIntegrationTest): hello_install_requires_dir = 'testprojects/src/python/python_distribution/hello_with_install_requires' hello_setup_requires = 'examples/src/python/example/python_distribution/hello/setup_requires' - py_dist_test = 'examples/tests/python/example_test/python_distribution' + py_dist_test = 'testprojects/tests/python/example_test/python_distribution' - def _assert_greeting(self, output): - self.assertIn('hello!', output) + def _assert_nation_and_greeting(self, output, punctuation='!'): + self.assertEquals("""\ +hello{} +United States +""".format(punctuation), output) - def test_pants_binary(self): + def test_pydist_binary(self): with temporary_dir() as tmp_dir: - pex = os.path.join(tmp_dir, 'main.pex') - command=[ - '--pants-distdir={}'.format(tmp_dir), 'binary', + pex = os.path.join(tmp_dir, 'main_with_no_conflict.pex') + command = [ + '--pants-distdir={}'.format(tmp_dir), + 'binary', '{}:main_with_no_conflict'.format(self.hello_install_requires_dir), ] pants_run = self.run_pants(command=command) @@ -38,22 +41,23 @@ def test_pants_binary(self): self.assertTrue(os.path.isfile(pex)) # Check that the pex runs. output = subprocess.check_output(pex).decode('utf-8') - self._assert_greeting(output) + self._assert_nation_and_greeting(output) # Check that we have exactly one wheel output. single_wheel_output = assert_single_element(glob.glob(os.path.join(tmp_dir, '*.whl'))) assertRegex(self, os.path.basename(single_wheel_output), - r'\A{}'.format(re.escape('hello_with_install_requires-1.0.0+'))) + r'\A{}'.format(re.escape('hello_with_install_requires-1.0.0+'))) - def test_pants_run(self): + def test_pydist_run(self): with temporary_dir() as tmp_dir: command=[ '--pants-distdir={}'.format(tmp_dir), + '--quiet', 'run', '{}:main_with_no_conflict'.format(self.hello_install_requires_dir)] pants_run = self.run_pants(command=command) self.assert_success(pants_run) # Check that text was properly printed to stdout. - self._assert_greeting(pants_run.stdout_data) + self._assert_nation_and_greeting(pants_run.stdout_data) def test_pydist_invalidation(self): """Test that the current version of a python_dist() is resolved after modifying its sources.""" @@ -69,22 +73,23 @@ def test_pydist_invalidation(self): unmodified_pants_run = run_target() self.assert_success(unmodified_pants_run) - self.assertEquals('hello!', unmodified_pants_run.stdout_data) + self._assert_nation_and_greeting(unmodified_pants_run.stdout_data) # Modify one of the source files for this target so that the output is different. py_source_file = os.path.join( self.hello_install_requires_dir, 'hello_package/hello.py') with open(py_source_file, 'r') as f: orig_contents = f.read() + # Replace hello! with hello? modified_contents = re.sub('!', '?', orig_contents) with open(py_source_file, 'w') as f: f.write(modified_contents) modified_pants_run = run_target() self.assert_success(modified_pants_run) - self.assertEquals('hello?', modified_pants_run.stdout_data) + self._assert_nation_and_greeting(modified_pants_run.stdout_data, punctuation='?') - def test_pants_test(self): + def test_pydist_test(self): with temporary_dir() as tmp_dir: command=[ '--pants-distdir={}'.format(tmp_dir), @@ -94,80 +99,4 @@ def test_pants_test(self): pants_run = self.run_pants(command=command) self.assert_success(pants_run) # Make sure that there is no wheel output when 'binary' goal is not invoked. - self.assertEqual(len(glob.glob(os.path.join(tmp_dir, '*.whl'))), 0) - - def test_with_install_requires(self): - with temporary_dir() as tmp_dir: - pex = os.path.join(tmp_dir, 'main_with_no_conflict.pex') - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'run', - '{}:main_with_no_conflict'.format(self.hello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - self.assertIn('United States', pants_run.stdout_data) - command=['binary', '{}:main_with_no_conflict'.format(self.hello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - output = subprocess.check_output(pex).decode('utf-8') - self.assertIn('United States', output) - - def test_with_conflicting_transitive_deps(self): - command=['run', '{}:main_with_conflicting_dep'.format(self.hello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_failure(pants_run) - self.assertIn('pycountry', pants_run.stderr_data) - self.assertIn('fasthello', pants_run.stderr_data) - command=['binary', '{}:main_with_conflicting_dep'.format(self.hello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_failure(pants_run) - self.assertIn('pycountry', pants_run.stderr_data) - self.assertIn('fasthello', pants_run.stderr_data) - - def test_binary_dep_isolation_with_multiple_targets(self): - with temporary_dir() as tmp_dir: - pex1 = os.path.join(tmp_dir, 'main_with_no_conflict.pex') - pex2 = os.path.join(tmp_dir, 'main_with_no_pycountry.pex') - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'binary', - '{}:main_with_no_conflict'.format(self.hello_install_requires_dir), - '{}:main_with_no_pycountry'.format(self.hello_install_requires_dir)] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - # Check that the pex was built. - self.assertTrue(os.path.isfile(pex1)) - self.assertTrue(os.path.isfile(pex2)) - # Check that the pex 1 runs. - output = subprocess.check_output(pex1).decode('utf-8') - self._assert_greeting(output) - # Check that the pex 2 fails due to no python_dists leaked into it. - try: - subprocess.check_output(pex2) - except subprocess.CalledProcessError as e: - self.assertNotEqual(0, e.returncode) - - def test_python_distribution_with_setup_requires(self): - # Validate that setup_requires dependencies are present and functional. - # PANTS_TEST_SETUP_REQUIRES triggers test functionality in this particular setup.py. - with environment_as(PANTS_TEST_SETUP_REQUIRES='1'): - command=['run', '{}:main'.format(self.hello_setup_requires)] - pants_run = self.run_pants(command=command) - - command=['run', '{}:main'.format(self.hello_setup_requires)] - pants_run = self.run_pants(command=command) - - with temporary_dir() as tmp_dir: - pex = os.path.join(tmp_dir, 'main.pex') - command=[ - '--pants-distdir={}'.format(tmp_dir), - 'binary', - '{}:main'.format(self.hello_setup_requires), - ] - pants_run = self.run_pants(command=command) - self.assert_success(pants_run) - # Check that the pex was built. - self.assertTrue(is_executable(pex)) - # Check that the pex runs. - output = subprocess.check_output(pex).decode('utf-8') - self.assertEqual('Hello, world!\n', output) + self.assertEqual(0, len(glob.glob(os.path.join(tmp_dir, '*.whl')))) diff --git a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py index 3831dbed46d7..09199db24421 100644 --- a/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py +++ b/tests/python/pants_test/backend/python/tasks/util/build_local_dists_test_base.py @@ -4,18 +4,23 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import os import re -from builtins import next +from builtins import next, str from pants.backend.native.register import rules as native_backend_rules from pants.backend.native.subsystems.libc_dev import LibcDev from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.tasks.build_local_python_distributions import \ BuildLocalPythonDistributions +from pants.backend.python.tasks.resolve_requirements import ResolveRequirements from pants.backend.python.tasks.select_interpreter import SelectInterpreter +from pants.build_graph.address import Address from pants.util.collections import assert_single_element -from pants.util.memo import memoized_property -from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase +from pants.util.memo import memoized_method +from pants.util.meta import classproperty +from pants_test.backend.python.tasks.python_task_test_base import (PythonTaskTestBase, + name_and_platform) from pants_test.engine.scheduler_test_base import SchedulerTestBase @@ -25,22 +30,37 @@ class BuildLocalPythonDistributionsTestBase(PythonTaskTestBase, SchedulerTestBas def task_type(cls): return BuildLocalPythonDistributions - # This is an informally-specified nested dict -- see ../test_ctypes.py for an example. Special - # keys are 'key' (used to index into `self.target_dict`) and 'filemap' (creates files at the - # specified relative paths). The rest of the keys are fed into `self.make_target()`. An - # `OrderedDict` of 2-tuples may be used if targets need to be created in a specific order (e.g. if - # they have dependencies on each other). - _dist_specs = None - # By default, we just use a `BuildLocalPythonDistributions` task. When testing with C/C++ targets, - # we want to compile and link them as well to get the resulting dist to build, so we add those - # task types here and execute them beforehand. - _extra_relevant_task_types = [SelectInterpreter] - - @memoized_property - def _all_other_synthesized_task_types(self): + @classproperty + def _dist_specs(cls): + """ + This is an informally-specified nested dict -- see ../test_ctypes.py for an example. Special + keys are 'key' (used to index into `self.target_dict`) and 'filemap' (creates files at the + specified relative paths). The rest of the keys are fed into `self.make_target()`. An + `OrderedDict` of 2-tuples may be used if targets need to be created in a specific order (e.g. if + they have dependencies on each other). + """ + raise NotImplementedError('_dist_specs must be implemented!') + + @classproperty + def _run_before_task_types(cls): + """ + By default, we just use a `BuildLocalPythonDistributions` task. When testing with C/C++ targets, + we want to compile and link them as well to get the resulting dist to build, so we add those + task types here and execute them beforehand. + """ + return [SelectInterpreter] + + @classproperty + def _run_after_task_types(cls): + """Tasks to run after local dists are built, similar to `_run_before_task_types`.""" + return [ResolveRequirements] + + @memoized_method + def _synthesize_task_types(self, task_types=()): return [ self.synthesize_task_subtype(tsk, '__tmp_{}'.format(tsk.__name__)) - for tsk in self._extra_relevant_task_types + # TODO: make @memoized_method convert lists to tuples for hashing! + for tsk in task_types ] def setUp(self): @@ -48,20 +68,31 @@ def setUp(self): self.target_dict = {} - # Create a python_dist() target from each specification and insert it into `self.target_dict`. - for target_spec, file_spec in self._dist_specs.items(): - file_spec = file_spec.copy() - filemap = file_spec.pop('filemap') + # Create a target from each specification and insert it into `self.target_dict`. + for target_spec, target_kwargs in self._dist_specs.items(): + unprocessed_kwargs = target_kwargs.copy() + + target_base = Address.parse(target_spec).spec_path + + # Populate the target's owned files from the specification. + filemap = unprocessed_kwargs.pop('filemap', {}) for rel_path, content in filemap.items(): - self.create_file(rel_path, content) + buildroot_path = os.path.join(target_base, rel_path) + self.create_file(buildroot_path, content) - key = file_spec.pop('key') + # Ensure any dependencies exist in the target dict (`_dist_specs` must then be an + # OrderedDict). + # The 'key' is used to access the target in `self.target_dict`. + key = unprocessed_kwargs.pop('key') dep_targets = [] - for dep_spec in file_spec.pop('dependencies', []): + for dep_spec in unprocessed_kwargs.pop('dependencies', []): existing_tgt_key = self._dist_specs[dep_spec]['key'] dep_targets.append(self.target_dict[existing_tgt_key]) - python_dist_tgt = self.make_target(spec=target_spec, dependencies=dep_targets, **file_spec) - self.target_dict[key] = python_dist_tgt + + # Register the generated target. + generated_target = self.make_target( + spec=target_spec, dependencies=dep_targets, **unprocessed_kwargs) + self.target_dict[key] = generated_target def _all_specified_targets(self): return list(self.target_dict.values()) @@ -105,15 +136,16 @@ def _create_task(self, task_type, context): return task_type(context, self.test_workdir) def _create_distribution_synthetic_target(self, python_dist_target, extra_targets=[]): - + run_before_synthesized_task_types = self._synthesize_task_types(tuple(self._run_before_task_types)) python_create_distributions_task_type = self._testing_task_type - all_synthesized_task_types = self._all_other_synthesized_task_types + [ + run_after_synthesized_task_types = self._synthesize_task_types(tuple(self._run_after_task_types)) + all_synthesized_task_types = run_before_synthesized_task_types + [ python_create_distributions_task_type, - ] + ] + run_after_synthesized_task_types context = self._scheduling_context( target_roots=([python_dist_target] + extra_targets), - for_task_types=(all_synthesized_task_types), + for_task_types=all_synthesized_task_types, for_subsystems=[PythonRepos, LibcDev], # TODO(#6848): we should be testing all of these with both of our toolchains. options={ @@ -123,19 +155,23 @@ def _create_distribution_synthetic_target(self, python_dist_target, extra_target }) self.assertEqual(set(self._all_specified_targets()), set(context.build_graph.targets())) - all_other_task_instances = [ + run_before_task_instances = [ self._create_task(task_type, context) - for task_type in self._all_other_synthesized_task_types + for task_type in run_before_synthesized_task_types ] python_create_distributions_task_instance = self._create_task( - python_create_distributions_task_type, - context) + python_create_distributions_task_type, context) + run_after_task_instances = [ + self._create_task(task_type, context) + for task_type in run_after_synthesized_task_types + ] + all_task_instances = run_before_task_instances + [ + python_create_distributions_task_instance + ] + run_after_task_instances - for tsk in all_other_task_instances: + for tsk in all_task_instances: tsk.execute() - python_create_distributions_task_instance.execute() - synthetic_tgts = set(context.build_graph.targets()) - set(self._all_specified_targets()) self.assertEqual(1, len(synthetic_tgts)) synthetic_target = next(iter(synthetic_tgts)) @@ -144,3 +180,20 @@ def _create_distribution_synthetic_target(self, python_dist_target, extra_target python_create_distributions_task_instance, python_dist_target) return context, synthetic_target, snapshot_version + + def _assert_dist_and_wheel_identity(self, expected_name, expected_version, expected_platform, + dist_target, **kwargs): + context, synthetic_target, fingerprint_suffix = self._create_distribution_synthetic_target( + dist_target, **kwargs) + resulting_dist_req = assert_single_element(synthetic_target.requirements.value) + expected_snapshot_version = '{}+{}'.format(expected_version, fingerprint_suffix) + self.assertEquals( + '{}=={}'.format(expected_name, expected_snapshot_version), + str(resulting_dist_req.requirement)) + + local_wheel_products = context.products.get('local_wheels') + local_wheel = self._retrieve_single_product_at_target_base(local_wheel_products, dist_target) + dist, version, platform = name_and_platform(local_wheel) + self.assertEquals(dist, expected_name) + self.assertEquals(version, expected_snapshot_version) + self.assertEquals(platform, expected_platform)