Let plugins declare @rules and move the native backend into contrib

build-support/bin/release.sh

@@ -124,7 +124,6 @@ function execute_packaged_pants_with_internal_backends() {
         'pants.backend.docgen',\
         'pants.backend.graph_info',\
         'pants.backend.jvm',\
-        'pants.backend.native',\
         'pants.backend.project_info',\
         'pants.backend.python',\
         'internal_backend.repositories',\

contrib/native/src/python/pants/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

contrib/native/src/python/pants/contrib/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

contrib/native/src/python/pants/contrib/native/BUILD

@@ -0,0 +1,14 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+contrib_plugin(
+  name='plugin',
+  dependencies=[
+    'contrib/native/src/python/pants/contrib/native/subsystems',
+    'contrib/native/src/python/pants/contrib/native/tasks',
+    'src/python/pants/goal:task_registrar',
+  ],
+  distribution_name='pantsbuild.pants.contrib.native',
+  description='C/C++ support for pants.',
+  register_goals=True,
+)

contrib/native/src/python/pants/contrib/native/register.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.goal.task_registrar import TaskRegistrar as task
+
+from pants.contrib.native.subsystems.native_toolchain import create_native_toolchain_rules
+from pants.contrib.native.tasks.populate_native_environment import PopulateNativeEnvironment
+
+
+def register_goals():
+  task(name='populate-native-environment', action=PopulateNativeEnvironment).install('bootstrap')
+
+
+def rules():
+  return create_native_toolchain_rules()

src/python/pants/backend/native/subsystems/BUILD → contrib/native/src/python/pants/contrib/native/subsystems/BUILD

@@ -3,9 +3,12 @@
 
 python_library(
   dependencies=[
-    'src/python/pants/backend/native/subsystems/platform_specific/darwin',
-    'src/python/pants/backend/native/subsystems/platform_specific/linux',
+    'contrib/native/src/python/pants/contrib/native/subsystems/platform_specific/darwin',
+    'contrib/native/src/python/pants/contrib/native/subsystems/platform_specific/linux',
+    'src/python/pants/backend/python/tasks',
     'src/python/pants/binaries',
+    'src/python/pants/engine:rules',
+    'src/python/pants/engine:selectors',
     'src/python/pants/subsystem',
     'src/python/pants/util:contextutil',
     'src/python/pants/util:memo',

src/python/pants/backend/native/subsystems/native_toolchain.py → contrib/native/src/python/pants/contrib/native/subsystems/native_toolchain.py

@@ -5,15 +5,19 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
-from pants.backend.native.subsystems.gcc import GCC
-from pants.backend.native.subsystems.llvm import LLVM
-from pants.backend.native.subsystems.platform_specific.darwin.xcode_cli_tools import XCodeCLITools
-from pants.backend.native.subsystems.platform_specific.linux.binutils import Binutils
+from pants.backend.python.tasks.setup_py import NativeToolchainEnvironment
 from pants.binaries.binary_tool import ExecutablePathProvider
+from pants.engine.rules import RootRule, rule
+from pants.engine.selectors import Select
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_method
 from pants.util.osutil import get_os_name, normalize_os_name
 
+from pants.contrib.native.subsystems.gcc import GCC
+from pants.contrib.native.subsystems.llvm import LLVM
+from pants.contrib.native.subsystems.platform_specific.darwin.xcode_cli_tools import XCodeCLITools
+from pants.contrib.native.subsystems.platform_specific.linux.binutils import Binutils
+
 
 class NativeToolchain(Subsystem, ExecutablePathProvider):
   """Abstraction over platform-specific tools to compile and link native code.
@@ -99,3 +103,15 @@ def path_entries(self):
       combined_path_entries.extend(subsystem.path_entries())
 
     return combined_path_entries
+
+
+@rule(NativeToolchainEnvironment, [Select(NativeToolchain)])
+def get_native_toolchain_environment(native_toolchain):
+  yield NativeToolchainEnvironment(tuple(native_toolchain.path_entries()))
+
+
+def create_native_toolchain_rules():
+  return [
+    RootRule(NativeToolchain),
+    get_native_toolchain_environment,
+  ]

contrib/native/src/python/pants/contrib/native/tasks/BUILD

@@ -0,0 +1,10 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies=[
+    'src/python/pants/backend/python/tasks',
+    'src/python/pants/task',
+    'src/python/pants/util:memo',
+  ],
+)

contrib/native/src/python/pants/contrib/native/tasks/populate_native_environment.py

@@ -0,0 +1,38 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.python.tasks.setup_py import NativeToolchainEnvironment
+from pants.task.task import Task
+from pants.util.memo import memoized_property
+
+from pants.contrib.native.subsystems.native_toolchain import NativeToolchain
+
+
+class PopulateNativeEnvironment(Task):
+
+  @classmethod
+  def product_types(cls):
+    return [NativeToolchainEnvironment]
+
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(PopulateNativeEnvironment, cls).subsystem_dependencies() + (
+      NativeToolchain.scoped(cls),
+    )
+
+  @memoized_property
+  def _native_toolchain(self):
+    return NativeToolchain.scoped_instance(self)
+
+  def _request_single(self, product, subject):
+    # This is not supposed to be exposed to Tasks yet -- see #4769 to track the
+    # status of exposing v2 products in v1 tasks.
+    return self.context._scheduler.product_request(product, [subject])[0]
+
+  def execute(self):
+    env = self._request_single(NativeToolchainEnvironment, self._native_toolchain)
+    self.context.products.register_data(NativeToolchainEnvironment, env)

contrib/native/tests/python/pants_test/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

contrib/native/tests/python/pants_test/contrib/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

tests/python/pants_test/backend/native/subsystems/BUILD → contrib/native/tests/python/pants_test/contrib/native/subsystems/BUILD

@@ -4,7 +4,7 @@
 
 python_tests(
   dependencies=[
-    'src/python/pants/backend/native/subsystems',
+    'contrib/native/src/python/pants/contrib/native/subsystems',
     'src/python/pants/util:contextutil',
     'src/python/pants/util:process_handler',
     'tests/python/pants_test:base_test',

tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py → contrib/native/tests/python/pants_test/contrib/native/subsystems/test_native_toolchain.py

@@ -7,12 +7,13 @@
 
 import unittest
 
-from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.util.contextutil import environment_as, get_joined_path
 from pants.util.process_handler import subprocess
 from pants_test.base_test import BaseTest
 from pants_test.subsystem.subsystem_util import global_subsystem_instance
 
+from pants.contrib.native.subsystems.native_toolchain import NativeToolchain
+
 
 # TODO(cosmicexplorer): Can we have some form of this run in an OSX shard on
 # Travis?

contrib/release_packages.sh

@@ -46,6 +46,18 @@ function pkg_go_install_test() {
       --explain test | grep "GoTest_test_go" &> /dev/null
 }
 
+PKG_NATIVE=(
+  "pantsbuild.pants.contrib.native"
+  "//contrib/native/src/python/pants/contrib/native:plugin"
+  "pkg_native_install_test"
+)
+function pkg_native_install_test() {
+  local version="$1"
+  execute_packaged_pants_with_internal_backends \
+    --plugins="['pantsbuild.pants.contrib.native==${version}']" \
+    --explain test | grep "NativeTest_test_native" &> /dev/null
+}
+
 PKG_NODE=(
   "pantsbuild.pants.contrib.node"
   "//contrib/node/src/python/pants/contrib/node:plugin"
@@ -217,6 +229,7 @@ CONTRIB_PACKAGES=(
   PKG_SCROOGE
   PKG_BUILDGEN
   PKG_GO
+  PKG_NATIVE
   PKG_NODE
   PKG_PYTHON_CHECKS
   PKG_SCALAJS

pants.ini

@@ -35,6 +35,7 @@ pythonpath: [
     "%(buildroot)s/contrib/googlejavaformat/src/python",
     "%(buildroot)s/contrib/jax_ws/src/python",
     "%(buildroot)s/contrib/mypy/src/python",
+    "%(buildroot)s/contrib/native/src/python",
     "%(buildroot)s/contrib/node/src/python",
     "%(buildroot)s/contrib/python/src/python",
     "%(buildroot)s/contrib/scalajs/src/python",
@@ -60,6 +61,7 @@ backend_packages: +[
     "pants.contrib.jax_ws",
     "pants.contrib.scalajs",
     "pants.contrib.mypy",
+    "pants.contrib.native",
     "pants.contrib.node",
     "pants.contrib.python.checks",
     "pants.contrib.scrooge",

src/python/pants/backend/python/tasks/BUILD

@@ -6,7 +6,6 @@ python_library(
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
     '3rdparty/python:pex',
-    'src/python/pants/backend/native/subsystems',
     'src/python/pants/backend/python/subsystems',
     'src/python/pants/backend/python/targets',
     'src/python/pants/backend/python/tasks/coverage:plugin',

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -13,20 +13,20 @@
 from pex.interpreter import PythonInterpreter
 from wheel.install import WheelFile
 
-from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.tasks.pex_build_util import _resolve_multi
-from pants.backend.python.tasks.setup_py import SetupPyInvocationEnvironment, SetupPyRunner
+from pants.backend.python.tasks.setup_py import (NativeToolchainEnvironment,
+                                                 SetupPyInvocationEnvironment, SetupPyRunner)
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TargetDefinitionException, TaskError
 from pants.base.fingerprint_strategy import DefaultFingerprintStrategy
 from pants.build_graph.address import Address
 from pants.task.task import Task
 from pants.util.contextutil import environment_as
 from pants.util.dirutil import safe_mkdir
-from pants.util.memo import memoized_method
+from pants.util.memo import memoized_property
 
 
 class BuildLocalPythonDistributions(Task):
@@ -44,19 +44,12 @@ def product_types(cls):
   @classmethod
   def prepare(cls, options, round_manager):
     round_manager.require_data(PythonInterpreter)
+    round_manager.optional_product(NativeToolchainEnvironment)
 
   @classmethod
   def implementation_version(cls):
     return super(BuildLocalPythonDistributions, cls).implementation_version() + [('BuildLocalPythonDistributions', 2)]
 
-  @classmethod
-  def subsystem_dependencies(cls):
-    return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + (NativeToolchain.scoped(cls),)
-
-  @memoized_method
-  def _native_toolchain_instance(self):
-    return NativeToolchain.scoped_instance(self)
-
   @property
   def cache_target_dirs(self):
     return True
@@ -146,6 +139,14 @@ def _request_single(self, product, subject):
     # status of exposing v2 products in v1 tasks.
     return self.context._scheduler.product_request(product, [subject])[0]
 
+  @memoized_property
+  def _native_toolchain_environment(self):
+    env = self.context.products.get_data(NativeToolchainEnvironment)
+    if not env:
+      env = NativeToolchainEnvironment(tuple())
+
+    return env
+
   # FIXME(cosmicexplorer): We should be isolating the path to just our provided
   # toolchain, but this causes errors in Travis because distutils looks for
   # "x86_64-linux-gnu-gcc" when linking native extensions. We almost definitely
@@ -155,7 +156,7 @@ def _request_single(self, product, subject):
   @contextmanager
   def _setup_py_invocation_environment(self, pythonpath):
     setup_py_env = self._request_single(
-      SetupPyInvocationEnvironment, self._native_toolchain_instance())
+      SetupPyInvocationEnvironment, self._native_toolchain_environment)
     env = setup_py_env.as_env_dict()
     if pythonpath:
       self.context.log.debug('Setting PYTHONPATH with setup_requires site directory: {}'

src/python/pants/backend/python/tasks/setup_py.py

@@ -19,7 +19,6 @@
 from twitter.common.collections import OrderedSet
 from twitter.common.dirutil.chroot import Chroot
 
-from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_target import PythonTarget
@@ -30,7 +29,7 @@
 from pants.build_graph.address_lookup_error import AddressLookupError
 from pants.build_graph.build_graph import sort_targets
 from pants.build_graph.resources import Resources
-from pants.engine.rules import rule
+from pants.engine.rules import RootRule, rule
 from pants.engine.selectors import Select
 from pants.task.task import Task
 from pants.util.contextutil import get_joined_path
@@ -79,6 +78,13 @@ def _setup_command(self):
     return self.__setup_command
 
 
+class NativeToolchainEnvironment(datatype([('path_entries', tuple)])):
+  """FIXME: this is a stub for now to facilitate moving the native backend to contrib in #5970.
+
+  This will be removed when #5815 is merged.
+  """
+
+
 class SetupPyInvocationEnvironment(datatype(['joined_path'])):
 
   def as_env_dict(self):
@@ -89,10 +95,9 @@ def as_env_dict(self):
     }
 
 
-@rule(SetupPyInvocationEnvironment, [Select(NativeToolchain)])
-def get_setup_py_env(native_toolchain):
-  joined_path = get_joined_path(
-    native_toolchain.path_entries(), os.environ.copy())
+@rule(SetupPyInvocationEnvironment, [Select(NativeToolchainEnvironment)])
+def get_setup_py_env(native_toolchain_environment):
+  joined_path = get_joined_path(native_toolchain_environment.path_entries, os.environ.copy())
   return SetupPyInvocationEnvironment(joined_path)
 
 
@@ -666,5 +671,6 @@ def create(exported_python_target):
 
 def create_setup_py_rules():
   return [
+    RootRule(NativeToolchainEnvironment),
     get_setup_py_env,
   ]

src/python/pants/engine/legacy/source_mapper.py

@@ -93,4 +93,3 @@ def iter_target_addresses_for_sources(self, sources):
       if (LegacyAddressMapper.any_is_declaring_file(legacy_address, sources_set) or
           self._owns_any_source(sources_set, hydrated_target)):
         yield legacy_address
-