Fix edge case when building packages with optional c extensions

The edge case occurs when a package can compile optional c speedups
which succeed during the build process but lock the generated wheel file
to the arch the package was built on. This makes the wheel file
incompatible with lambda and causes it to fail to add that dependency
even though it could have by building it without the optional c
extensions.

This is a bit tricky to fix as means we need to selectivly break the
ability to compile c extensions during some runs of our wheel buiding
phase. It is also quite differnet on POSIX and Windows.

chalice/compat.py

@@ -1,8 +1,112 @@
 import socket
 import six
+import os
 
 from six import StringIO
 
+if os.name == 'nt':
+    # windows
+    # On windows running python in a subprocess with no environment variables
+    # will cause sevral issues. In order for our subprocess to run normally we
+    # manually copy the relevant environment variables from the parent process.
+    subprocess_python_base_environ = {
+        # http://bugs.python.org/issue8557
+        'PATH': os.environ['PATH']
+    }
+    # http://bugs.python.org/issue20614
+    if 'SYSTEMROOT' in os.environ:
+        subprocess_python_base_environ['SYSTEMROOT'] = os.environ['SYSTEMROOT']
+
+    # This is the acutal patch used on windows to prevent distutils from
+    # compiling C extensions. The msvc compiler base class has its compile
+    # method overridden to raise a CompileError. This can be caught by
+    # setup.py code which can then fallback to making a purepython
+    # package if possible.
+    # We need mypy to ignore these since they are never actually called from
+    # within ourprocess they do not need to be a part of our typechecking
+    # pass.
+    def prevent_msvc_compiling_patch():  # type: ignore
+        import distutils
+        import distutils._msvccompiler
+        import distutils.msvc9compiler
+        import distutils.msvccompiler
+
+        from distutils.errors import CompileError
+
+        def raise_compile_error(*args, **kwargs):  # type: ignore
+            raise CompileError('Chalice blocked C extension compiling.')
+        distutils._msvccompiler.MSVCCompiler.compile = raise_compile_error
+        distutils.msvc9compiler.MSVCCompiler.compile = raise_compile_error
+        distutils.msvccompiler.MSVCCompiler.compile = raise_compile_error
+
+    # This is the setuptools shim used to execute setup.py by pip.
+    # Lines 2 and 3 have been added to call the above function
+    # `prevent_msvc_compiling_patch` and extra escapes have been added on line
+    # 5 because it is passed through another layer of string parsing before it
+    # is executed.
+    _SETUPTOOLS_SHIM = (
+        "import setuptools, tokenize;__file__=%r;"
+        "from chalice.compat import prevent_msvc_compiling_patch;"
+        "prevent_msvc_compiling_patch();"
+        "f=getattr(tokenize, 'open', open)(__file__);"
+        "code=f.read().replace('\\\\r\\\\n', '\\\\n');"
+        "f.close();"
+        "exec(compile(code, __file__, 'exec'))"
+    )
+
+    # On windows the C compiling story is much more complex than on posix as
+    # there are many different C compilers that setuptools and disutils will
+    # try and find using a combination of known filepaths, registry entries,
+    # and environment variables. Since there is no simple environment variable
+    # we can replace when starting the subprocess that builds the package;
+    # we need to apply a patch at runtime to prevent pip/setuptools/distutils
+    # from being able to build C extensions.
+    # Patching out every possible technique for finding each compiler would
+    # be a losing game of whack-a-mole. In addition we need to apply a patch
+    # two layers down through subprocess calls, specifically:
+    #  * Chalice creates a subprocess of `pip wheel ...` to build sdists
+    #    into wheel files.
+    #  * Pip creates another python subprocess to call the setup.py file in
+    #    the sdist. Before doing so it applies the above shim to make the
+    #    setup file compatible with setuptools. This shim layer also reads
+    #    and executes the code in the setup.py.
+    #  * Setuptools (which will have been executed by the shim) will
+    #    eventually call distutils to do the heavy lifting for C compiling.
+    #
+    # Our patch needs to affect the bottom level here (distutils) and patch
+    # it out to prevent it from compiling C in a graceful way that results in
+    # falling back to building a purepython library if possible.
+    # The below line will be injected just before the `pip wheel ...` portion
+    # of the subprocess that Chalice starts. This replaces the
+    # SETUPTOOLS_SHIM that pip normally uses with the one defined above.
+    # When pip goes to run its subprocess for executing setup.py it will
+    # inject _SETUPTOOLS_SHIM rather than the usual SETUPTOOLS_SHIM in pip.
+    # This lets us apply our patches in the same process that will compile
+    # the c extensions before the setup.py file has been executed.
+    # The actual patches used are decribed in the comment above
+    # _SETUPTOOLS_SHIM.
+    pip_no_compile_c_shim = (
+        "import pip;"
+        "pip.wheel.SETUPTOOLS_SHIM = \"\"\"%s\"\"\";"
+    ) % _SETUPTOOLS_SHIM
+    pip_no_compile_c_env_vars = {}  # type: ignore
+else:
+    # posix
+    # On posix you can start python in a subprocess with no environment
+    # variables and it will run normally.
+    subprocess_python_base_environ = {}
+
+    # On posix systems setuptools/distutils uses the CC env variable to
+    # locate a C compiler for building C extensions. All we need to do is set
+    # it to /var/false, and the module building process will fail to build.
+    # C extensions, and any fallback processes in place to build a pure python
+    # package will be kicked off.
+    # No need to monkey patch the process.
+    pip_no_compile_c_shim = ''
+    pip_no_compile_c_env_vars = {
+        'CC': '/var/false'
+    }
+
 
 if six.PY3:
     from urllib.parse import urlparse, parse_qs

chalice/deploy/packager.py

@@ -9,14 +9,24 @@
 from zipfile import ZipFile  # noqa
 
 from typing import Any, Set, List, Optional, Tuple, Iterable, Callable  # noqa
+from typing import Dict  # noqa
 from chalice.compat import lambda_abi
+from chalice.compat import subprocess_python_base_environ
+from chalice.compat import pip_no_compile_c_env_vars
+from chalice.compat import pip_no_compile_c_shim
 from chalice.utils import OSUtils
 from chalice.constants import MISSING_DEPENDENCIES_TEMPLATE
 
 import chalice
 from chalice import app
 
 
+StrMap = Dict[str, Any]
+OptStrMap = Optional[StrMap]
+OptStr = Optional[str]
+OptBytes = Optional[bytes]
+
+
 class InvalidSourceDistributionNameError(Exception):
     pass
 
@@ -313,11 +323,11 @@ def _download_binary_wheels(self, packages, directory):
         self._pip.download_manylinux_wheels(
             [pkg.identifier for pkg in packages], directory)
 
-    def _build_sdists(self, sdists, directory):
-        # type: (set[Package], str) -> None
+    def _build_sdists(self, sdists, directory, compile_c=True):
+        # type: (set[Package], str, bool) -> None
         for sdist in sdists:
             path_to_sdist = self._osutils.joinpath(directory, sdist.filename)
-            self._pip.build_wheel(path_to_sdist, directory)
+            self._pip.build_wheel(path_to_sdist, directory, compile_c)
 
     def _categorize_wheel_files(self, directory):
         # type: (str) -> Tuple[Set[Package], Set[Package]]
@@ -374,12 +384,25 @@ def _download_dependencies(self, directory, requirements_filename):
 
         # Re-count the wheel files after the second download pass. Anything
         # that has an sdist but not a valid wheel file is still not going to
-        # work on lambda and our last ditch effort is to try and build the
-        # sdists into wheel files.
+        # work on lambda and our we must now try and build the sdist into a
+        # wheel file ourselves.
+        compatible_wheels, incompatible_wheels = self._categorize_wheel_files(
+            directory)
+        missing_wheels = sdists - compatible_wheels
+        self._build_sdists(missing_wheels, directory, compile_c=True)
+
+        # There is still the case where the package had optional C dependencies
+        # for speedups. In this case the wheel file will have built above with
+        # the C dependencies if it managed to find a C compiler. If we are on
+        # an incompatible architecture this means the wheel file generated will
+        # not be compatible. If we categorize our files once more and find that
+        # there are missing dependencies we can try our last ditch effort of
+        # building the package and trying to sever its ability to find a C
+        # compiler.
         compatible_wheels, incompatible_wheels = self._categorize_wheel_files(
             directory)
         missing_wheels = sdists - compatible_wheels
-        self._build_sdists(missing_wheels, directory)
+        self._build_sdists(missing_wheels, directory, compile_c=False)
 
         # Final pass to find the compatible wheel files and see if there are
         # any unmet dependencies left over. At this point there is nothing we
@@ -555,13 +578,20 @@ def get_package_name_and_version(self, sdist_path):
 
 class SubprocessPip(object):
     """Wrapper around calling pip through a subprocess."""
-    def main(self, args):
-        # type: (List[str]) -> Tuple[int, Optional[bytes]]
+    def main(self, args, env_vars=None, shim=None):
+        # type: (List[str], OptStrMap, OptStr) -> Tuple[int, Optional[bytes]]
+        if env_vars is None:
+            env_vars = {}
+        if shim is None:
+            shim = ''
+        env_vars.update(subprocess_python_base_environ)
         python_exe = sys.executable
-        invoke_pip = [python_exe, '-m', 'pip']
-        invoke_pip.extend(args)
+        run_pip = 'import pip; pip.main(%s)' % args
+        exec_string = '%s%s' % (shim, run_pip)
+        invoke_pip = [python_exe, '-c', exec_string]
         p = subprocess.Popen(invoke_pip,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                             env=env_vars)
         _, err = p.communicate()
         rc = p.returncode
         return rc, err
@@ -573,21 +603,28 @@ def __init__(self, pip):
         # type: (SubprocessPip) -> None
         self._wrapped_pip = pip
 
-    def _execute(self, command, args):
-        # type: (str, List[str]) -> Tuple[int, Optional[bytes]]
+    def _execute(self, command, args, env_vars=None, shim=None):
+        # type: (str, List[str], OptStrMap, OptStr) -> Tuple[int, OptBytes]
         """Execute a pip command with the given arguments."""
         main_args = [command] + args
-        rc, err = self._wrapped_pip.main(main_args)
+        rc, err = self._wrapped_pip.main(main_args, env_vars=env_vars,
+                                         shim=shim)
         return rc, err
 
-    def build_wheel(self, wheel, directory):
-        # type: (str, str) -> None
+    def build_wheel(self, wheel, directory, compile_c=True):
+        # type: (str, str, bool) -> None
         """Build an sdist into a wheel file."""
         arguments = ['--no-deps', '--wheel-dir', directory, wheel]
+        env_vars = {}  # type: StrMap
+        shim = ''
+        if not compile_c:
+            env_vars.update(pip_no_compile_c_env_vars)
+            shim = pip_no_compile_c_shim
         # Ignore rc and stderr from this command since building the wheels
         # may fail and we will find out when we categorize the files that were
         # generated.
-        self._execute('wheel', arguments)
+        self._execute('wheel', arguments,
+                      env_vars=env_vars, shim=shim)
 
     def download_all_dependencies(self, requirements_filename, directory):
         # type: (str, str) -> None

tests/conftest.py

@@ -1,10 +1,14 @@
 import zipfile
+from collections import namedtuple
 
 import botocore.session
 from botocore.stub import Stubber
 from pytest import fixture
 
 
+FakePipCall = namedtuple('FakePipEntry', ['args', 'env_vars', 'shim'])
+
+
 class FakeSdistBuilder(object):
     _SETUP_PY = (
         'from setuptools import setup\n'