move python_dist() integration tests to unit tests, mostly

build-support/known_py3_pex_failures.txt

@@ -3,8 +3,6 @@ tests/python/pants_test/backend/jvm/tasks:jvm_platform_analysis_integration
 tests/python/pants_test/backend/jvm/tasks:scala_repl_integration
 tests/python/pants_test/backend/project_info/tasks:idea_plugin_integration
 tests/python/pants_test/backend/python/tasks:integration
-tests/python/pants_test/backend/python/tasks:isort_run_integration
-tests/python/pants_test/backend/python/tasks:python_native_code_testing_1
 tests/python/pants_test/backend/python:integration
 tests/python/pants_test/base:exiter_integration
 tests/python/pants_test/engine/legacy:changed_integration

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -7,8 +7,6 @@
 from builtins import str
 from collections import defaultdict
 
-from pex.pex import PEX
-
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.targets.native_library import NativeLibrary
 from pants.backend.python.python_requirement import PythonRequirement
@@ -19,7 +17,7 @@
 from pants.binaries.executable_pex_tool import ExecutablePexTool
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_property
-from pants.util.objects import SubclassesOf, datatype
+from pants.util.objects import SubclassesOf
 
 
 class PythonNativeCode(Subsystem):
@@ -151,9 +149,3 @@ def base_requirements(self):
       PythonRequirement('setuptools=={}'.format(self.python_setup.setuptools_version)),
       PythonRequirement('wheel=={}'.format(self.python_setup.wheel_version)),
     ]
-
-
-class SetupPyExecutionEnvironment(datatype([
-    # If None, don't set PYTHONPATH in the setup.py environment.
-    ('setup_requires_pex', PEX),
-])): pass

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -17,8 +17,7 @@
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.subsystems.pex_build_util import is_local_python_dist
 from pants.backend.python.subsystems.python_native_code import (BuildSetupRequiresPex,
-                                                                PythonNativeCode,
-                                                                SetupPyExecutionEnvironment)
+                                                                PythonNativeCode)
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TargetDefinitionException, TaskError
@@ -32,6 +31,7 @@
 from pants.util.strutil import safe_shlex_join
 
 
+# TODO: make this a SimpleCodegenTask!!!
 class BuildLocalPythonDistributions(Task):
   """Create python distributions (.whl) from python_dist targets."""
 
@@ -212,18 +212,15 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t
     setup_reqs_pex_path = os.path.join(
       setup_requires_dir,
       'setup-requires-{}.pex'.format(versioned_target_fingerprint))
-    extra_reqs = list(setup_reqs_to_resolve or [])
     setup_requires_pex = self._build_setup_requires_pex_settings.bootstrap(
-      interpreter, setup_reqs_pex_path, extra_reqs=extra_reqs)
+      interpreter, setup_reqs_pex_path, extra_reqs=setup_reqs_to_resolve)
     self.context.log.debug('Using pex file as setup.py interpreter: {}'
                            .format(setup_requires_pex))
 
-    setup_py_execution_environment = SetupPyExecutionEnvironment(setup_requires_pex)
-
     self._create_dist(
       dist_target,
       dist_output_dir,
-      setup_py_execution_environment,
+      setup_requires_pex,
       versioned_target_fingerprint,
       is_platform_specific)
 
@@ -256,7 +253,7 @@ def _generate_snapshot_bdist_wheel_argv(self, snapshot_fingerprint, is_platform_
   def _create_dist(self,
                    dist_tgt,
                    dist_target_dir,
-                   setup_py_execution_environment,
+                   setup_requires_pex,
                    snapshot_fingerprint,
                    is_platform_specific):
     """Create a .whl file for the specified python_distribution target."""
@@ -265,8 +262,6 @@ def _create_dist(self,
     setup_py_snapshot_version_argv = self._generate_snapshot_bdist_wheel_argv(
       snapshot_fingerprint, is_platform_specific)
 
-    setup_requires_pex = setup_py_execution_environment.setup_requires_pex
-
     cmd = safe_shlex_join(setup_requires_pex.cmdline(setup_py_snapshot_version_argv))
     with self.context.new_workunit('setup.py', cmd=cmd, labels=[WorkUnitLabel.TOOL]) as workunit:
       with pushd(dist_target_dir):
@@ -277,7 +272,7 @@ def _create_dist(self,
           raise self.BuildLocalPythonDistributionsError(
             "Installation of python distribution from target {target} into directory {into_dir} "
             "failed (return value of run() was: {rc!r}).\n"
-            "The chosen interpreter was: {interpreter}.\n"
+            "The pex with any requirements is located at: {interpreter}.\n"
             "The host system's compiler and linker were used.\n"
             "The setup command was: {command}."
             .format(target=dist_tgt,
@@ -307,7 +302,9 @@ def _get_whl_from_dir(cls, install_dir):
     dists = glob.glob(os.path.join(dist_dir, '*.whl'))
     if len(dists) == 0:
       raise cls.BuildLocalPythonDistributionsError(
-        'No distributions were produced by python_create_distribution task.')
+        'No distributions were produced by python_create_distribution task.\n'
+        'dist_dir: {}, install_dir: {}'
+        .format(dist_dir, install_dir))
     if len(dists) > 1:
       # TODO: is this ever going to happen?
       raise cls.BuildLocalPythonDistributionsError('Ambiguous local python distributions found: {}'

src/python/pants/backend/python/tasks/resolve_requirements.py

@@ -14,6 +14,8 @@ class ResolveRequirements(ResolveRequirementsTaskBase):
   """Resolve external Python requirements."""
   REQUIREMENTS_PEX = 'python_requirements_pex'
 
+  options_scope = 'resolve-requirements'
+
   @classmethod
   def product_types(cls):
     return [cls.REQUIREMENTS_PEX]

testprojects/src/python/python_distribution/hello_with_install_requires/BUILD

@@ -33,11 +33,3 @@ python_requirement_library(
     python_requirement('pycountry==17.9.23'),
   ]
 )
-
-python_binary(
-  name='main_with_no_pycountry',
-  source='main.py',
-  dependencies=[
-    ':hello_with_install_requires'
-  ],
-)

examples/tests/python/example_test/python_distribution/test_hello.py → testprojects/tests/python/example_test/python_distribution/test_hello.py

@@ -6,10 +6,10 @@
 
 import unittest
 
-from hello import hello_string
+from hello_package import hello
 
 
 class HelloTest(unittest.TestCase):
 
   def test_hello_import(self):
-    self.assertEqual('hello!', hello_string())
+    self.assertEqual('hello!', hello.hello_string())

tests/python/pants_test/backend/python/tasks/BUILD

@@ -22,85 +22,8 @@ python_library(
   ]
 )
 
-python_native_code_test_files = [
-  'test_build_local_python_distributions.py',
-  'test_build_local_python_distributions_integration.py',
-  'test_ctypes.py',
-  'test_ctypes_integration.py',
-]
-
-python_tests(
-  name='python_native_code_testing_3',
-  sources=[
-    python_native_code_test_files[0],
-  ],
-  dependencies=[
-    '3rdparty/python:future',
-    '3rdparty/python/twitter/commons:twitter.common.collections',
-    'src/python/pants/backend/native',
-    'src/python/pants/backend/native/targets',
-    'src/python/pants/backend/python:plugin',
-    'src/python/pants/backend/python/targets',
-    'src/python/pants/backend/python/tasks',
-    'src/python/pants/util:contextutil',
-    'src/python/pants/util:process_handler',
-    'tests/python/pants_test:int-test',
-    'tests/python/pants_test/backend/python/tasks/util',
-    'tests/python/pants_test/engine:scheduler_test_base',
-    'tests/python/pants_test/testutils:py2_compat',
-  ],
-  tags={'platform_specific_behavior', 'integration'},
-  timeout=2400,
-)
-
-python_tests(
-  name='python_native_code_testing_2',
-  sources=[
-    python_native_code_test_files[1],
-  ],
-  dependencies=[
-    '3rdparty/python:future',
-    '3rdparty/python/twitter/commons:twitter.common.collections',
-    'src/python/pants/backend/native',
-    'src/python/pants/backend/native/targets',
-    'src/python/pants/backend/python:plugin',
-    'src/python/pants/backend/python/targets',
-    'src/python/pants/backend/python/tasks',
-    'src/python/pants/util:contextutil',
-    'src/python/pants/util:process_handler',
-    'tests/python/pants_test:int-test',
-    'tests/python/pants_test/backend/python/tasks/util',
-    'tests/python/pants_test/engine:scheduler_test_base',
-    'tests/python/pants_test/testutils:py2_compat',
-  ],
-  tags={'platform_specific_behavior', 'integration'},
-  timeout=2400,
-)
-
 python_tests(
-  name='python_native_code_testing_1',
-  sources=python_native_code_test_files[2:],
-  dependencies=[
-    '3rdparty/python:future',
-    '3rdparty/python/twitter/commons:twitter.common.collections',
-    'src/python/pants/backend/native',
-    'src/python/pants/backend/native/targets',
-    'src/python/pants/backend/python:plugin',
-    'src/python/pants/backend/python/targets',
-    'src/python/pants/backend/python/tasks',
-    'src/python/pants/util:contextutil',
-    'src/python/pants/util:process_handler',
-    'tests/python/pants_test:int-test',
-    'tests/python/pants_test/backend/python/tasks/util',
-    'tests/python/pants_test/engine:scheduler_test_base',
-    'tests/python/pants_test/testutils:py2_compat',
-  ],
-  tags={'platform_specific_behavior', 'integration'},
-  timeout=2400,
-)
-
-python_tests(
-  sources=globs('test_*.py', exclude=([globs('*_integration.py')] + python_native_code_test_files)),
+  sources=globs('test_*.py', exclude=([globs('*_integration.py')])),
   dependencies=[
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
@@ -128,6 +51,7 @@ python_tests(
     'src/python/pants/util:process_handler',
     'src/python/pants/util:py2_compat',
     'tests/python/pants_test/backend/python:interpreter_selection_utils',
+    'tests/python/pants_test/backend/python/tasks/util',
     'tests/python/pants_test/engine:scheduler_test_base',
     'tests/python/pants_test/subsystem:subsystem_utils',
     'tests/python/pants_test/tasks:task_test_base',
@@ -139,7 +63,7 @@ python_tests(
 
 python_tests(
   name='integration',
-  sources=globs('*_integration.py', exclude=python_native_code_test_files),
+  sources=globs('*_integration.py'),
   dependencies=[
     '3rdparty/python:pex',
     'src/python/pants/base:build_environment',
@@ -154,28 +78,3 @@ python_tests(
   tags={'integration'},
   timeout=2400
 )
-
-python_tests(
-  name='isort_run_integration',
-  sources=['test_isort_run_integration.py'],
-  dependencies=[
-    'tests/python/pants_test:int-test',
-    'tests/python/pants_test/backend/python/tasks:python_task_test_base',
-  ],
-  tags={'integration'},
-)
-
-python_tests(
-  name='isort_run',
-  sources=['test_isort_run.py'],
-  dependencies=[
-    '3rdparty/python:future',
-    'src/python/pants/util:dirutil',
-    'tests/python/pants_test/backend/python/tasks:python_task_test_base',
-    'src/python/pants/util:process_handler',
-    'tests/python/pants_test:test_base',
-    'tests/python/pants_test/subsystem:subsystem_utils',
-    'tests/python/pants_test:task_test_base',
-  ],
-  timeout=300
-)