feat: warm containers (#2383)

requirements/base.txt

@@ -15,3 +15,4 @@ requests==2.23.0
 serverlessrepo==0.1.10
 aws_lambda_builders==1.1.0
 tomlkit==0.7.0
+watchdog==0.10.3

requirements/reproducible-linux.txt

@@ -134,6 +134,9 @@ markupsafe==1.1.1 \
     --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
     --hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be \
     # via cookiecutter, jinja2
+pathtools==0.1.2 \
+    --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 \
+    # via watchdog
 poyo==0.5.0 \
     --hash=sha256:3e2ca8e33fdc3c411cd101ca395668395dd5dc7ac775b8e809e3def9f9fe041a \
     --hash=sha256:e26956aa780c45f011ca9886f044590e2d8fd8b61db7b1c1cf4e0869f48ed4dd \
@@ -220,6 +223,9 @@ urllib3==1.25.8 \
     --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \
     --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc \
     # via botocore, requests
+watchdog==0.10.3 \
+    --hash=sha256:4214e1379d128b0588021880ccaf40317ee156d4603ac388b9adcf29165e0c04 \
+    # via aws-sam-cli (setup.py)
 websocket-client==0.57.0 \
     --hash=sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549 \
     --hash=sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010 \

samcli/commands/exceptions.py

@@ -53,3 +53,9 @@ class LambdaImagesTemplateException(UserException):
     """
     Exception class when multiple Lambda Image app templates are found for any runtime
     """
+
+
+class ContainersInitializationException(UserException):
+    """
+    Exception class when SAM is not able to initialize any of the lambda functions containers
+    """

samcli/commands/local/cli_common/invoke_context.py

@@ -1,23 +1,38 @@
 """
 Reads CLI arguments and performs necessary preparation to be able to run the function
 """
-
 import errno
 import json
+import logging
 import os
+from enum import Enum
 from pathlib import Path
 
 import samcli.lib.utils.osutils as osutils
+from samcli.lib.utils.async_utils import AsyncContext
 from samcli.lib.utils.stream_writer import StreamWriter
 from samcli.commands.local.lib.local_lambda import LocalLambdaRunner
 from samcli.commands.local.lib.debug_context import DebugContext
-from samcli.local.lambdafn.runtime import LambdaRuntime
+from samcli.local.lambdafn.runtime import LambdaRuntime, WarmLambdaRuntime
 from samcli.local.docker.lambda_image import LambdaImage
 from samcli.local.docker.manager import ContainerManager
 from samcli.commands._utils.template import get_template_data, TemplateNotFoundException, TemplateFailedParsingException
 from samcli.local.layers.layer_downloader import LayerDownloader
 from samcli.lib.providers.sam_function_provider import SamFunctionProvider
 from .user_exceptions import InvokeContextException, DebugContextException
+from ...exceptions import ContainersInitializationException
+
+LOG = logging.getLogger(__name__)
+
+
+class ContainersInitializationMode(Enum):
+    EAGER = "EAGER"
+    LAZY = "LAZY"
+
+
+class ContainersMode(Enum):
+    WARM = "WARM"
+    COLD = "COLD"
 
 
 class InvokeContext:
@@ -54,6 +69,8 @@ def __init__(
         force_image_build=None,
         aws_region=None,
         aws_profile=None,
+        warm_container_initialization_mode=None,
+        debug_function=None,
     ):
         """
         Initialize the context
@@ -91,6 +108,14 @@ def __init__(
             Whether or not to force build the image
         aws_region str
             AWS region to use
+        warm_container_initialization_mode str
+            Specifies how SAM cli manages the containers when using start-api or start_lambda.
+            Two modes are available:
+            "EAGER": Containers for every function are loaded at startup and persist between invocations.
+            "LAZY": Containers are only loaded when the function is first invoked and persist for additional invocations
+        debug_function str
+            The Lambda function logicalId that will have the debugging options enabled in case of warm containers
+            option is enabled
         """
         self._template_file = template_file
         self._function_identifier = function_identifier
@@ -109,6 +134,15 @@ def __init__(
         self._aws_region = aws_region
         self._aws_profile = aws_profile
 
+        self._containers_mode = ContainersMode.COLD
+        self._containers_initializing_mode = ContainersInitializationMode.LAZY
+
+        if warm_container_initialization_mode:
+            self._containers_mode = ContainersMode.WARM
+            self._containers_initializing_mode = ContainersInitializationMode(warm_container_initialization_mode)
+
+        self._debug_function = debug_function
+
         self._template_dict = None
         self._function_provider = None
         self._env_vars_value = None
@@ -117,6 +151,9 @@ def __init__(
         self._debug_context = None
         self._layers_downloader = None
         self._container_manager = None
+        self._lambda_runtimes = None
+
+        self._local_lambda_runner = None
 
     def __enter__(self):
         """
@@ -133,8 +170,27 @@ def __enter__(self):
         self._container_env_vars_value = self._get_env_vars_value(self._container_env_vars_file)
         self._log_file_handle = self._setup_log_file(self._log_file)
 
+        # in case of warm containers && debugging is enabled && if debug-function property is not provided, so
+        # if the provided template only contains one lambda function, so debug-function will be set to this function
+        # if the template contains multiple functions, a warning message "that the debugging option will be ignored"
+        # will be printed
+        if self._containers_mode == ContainersMode.WARM and self._debug_ports and not self._debug_function:
+            if len(self._function_provider.functions) == 1:
+                self._debug_function = list(self._function_provider.functions.keys())[0]
+            else:
+                LOG.info(
+                    "Warning: you supplied debugging options but you did not specify the --debug-function option."
+                    " To specify which function you want to debug, please use the --debug-function <function-name>"
+                )
+                # skipp the debugging
+                self._debug_ports = None
+
         self._debug_context = self._get_debug_context(
-            self._debug_ports, self._debug_args, self._debugger_path, self._container_env_vars_value
+            self._debug_ports,
+            self._debug_args,
+            self._debugger_path,
+            self._container_env_vars_value,
+            self._debug_function,
         )
 
         self._container_manager = self._get_container_manager(self._docker_network, self._skip_pull_image)
@@ -144,17 +200,56 @@ def __enter__(self):
                 "Running AWS SAM projects locally requires Docker. Have you got it installed and running?"
             )
 
+        # initialize all lambda function containers upfront
+        if self._containers_initializing_mode == ContainersInitializationMode.EAGER:
+            self._initialize_all_functions_containers()
+
         return self
 
     def __exit__(self, *args):
         """
-        Cleanup any necessary opened files
+        Cleanup any necessary opened resources
         """
 
         if self._log_file_handle:
             self._log_file_handle.close()
             self._log_file_handle = None
 
+        if self._containers_mode == ContainersMode.WARM:
+            self._clean_running_containers_and_related_resources()
+
+    def _initialize_all_functions_containers(self):
+        """
+        Create and run a container for each available lambda function
+        """
+        LOG.info("Initializing the lambda functions containers.")
+
+        def initialize_function_container(function):
+            function_config = self.local_lambda_runner.get_invoke_config(function)
+            self.lambda_runtime.run(None, function_config, self._debug_context, None)
+
+        try:
+            async_context = AsyncContext()
+            for function in self._function_provider.get_all():
+                async_context.add_async_task(initialize_function_container, function)
+
+            async_context.run_async(default_executor=False)
+            LOG.info("Containers Initialization is done.")
+        except KeyboardInterrupt:
+            LOG.debug("Ctrl+C was pressed. Aborting containers initialization")
+            self._clean_running_containers_and_related_resources()
+            raise
+        except Exception as ex:
+            LOG.error("Lambda functions containers initialization failed because of %s", ex)
+            self._clean_running_containers_and_related_resources()
+            raise ContainersInitializationException("Lambda functions containers initialization failed") from ex
+
+    def _clean_running_containers_and_related_resources(self):
+        """
+        Clean the running containers and any other related open resources
+        """
+        self.lambda_runtime.clean_running_containers_and_related_resources()
+
     @property
     def function_name(self):
         """
@@ -183,6 +278,18 @@ def function_name(self):
             "Possible options in your template: {}".format(all_function_names)
         )
 
+    @property
+    def lambda_runtime(self):
+        if not self._lambda_runtimes:
+            layer_downloader = LayerDownloader(self._layer_cache_basedir, self.get_cwd())
+            image_builder = LambdaImage(layer_downloader, self._skip_pull_image, self._force_image_build)
+            self._lambda_runtimes = {
+                ContainersMode.WARM: WarmLambdaRuntime(self._container_manager, image_builder),
+                ContainersMode.COLD: LambdaRuntime(self._container_manager, image_builder),
+            }
+
+        return self._lambda_runtimes[self._containers_mode]
+
     @property
     def local_lambda_runner(self):
         """
@@ -191,20 +298,19 @@ def local_lambda_runner(self):
         :return samcli.commands.local.lib.local_lambda.LocalLambdaRunner: Runner configured to run Lambda functions
             locally
         """
+        if self._local_lambda_runner:
+            return self._local_lambda_runner
 
-        layer_downloader = LayerDownloader(self._layer_cache_basedir, self.get_cwd())
-        image_builder = LambdaImage(layer_downloader, self._skip_pull_image, self._force_image_build)
-
-        lambda_runtime = LambdaRuntime(self._container_manager, image_builder)
-        return LocalLambdaRunner(
-            local_runtime=lambda_runtime,
+        self._local_lambda_runner = LocalLambdaRunner(
+            local_runtime=self.lambda_runtime,
             function_provider=self._function_provider,
             cwd=self.get_cwd(),
             aws_profile=self._aws_profile,
             aws_region=self._aws_region,
             env_vars_values=self._env_vars_value,
             debug_context=self._debug_context,
         )
+        return self._local_lambda_runner
 
     @property
     def stdout(self):
@@ -322,7 +428,7 @@ def _setup_log_file(log_file):
         return open(log_file, "wb")
 
     @staticmethod
-    def _get_debug_context(debug_ports, debug_args, debugger_path, container_env_vars):
+    def _get_debug_context(debug_ports, debug_args, debugger_path, container_env_vars, debug_function=None):
         """
         Creates a DebugContext if the InvokeContext is in a debugging mode
 
@@ -336,6 +442,9 @@ def _get_debug_context(debug_ports, debug_args, debugger_path, container_env_var
             Path to the directory of the debugger to mount on Docker
         container_env_vars dict
             Dictionary containing debugging based environmental variables.
+        debug_function str
+            The Lambda function logicalId that will have the debugging options enabled in case of warm containers
+            option is enabled
 
         Returns
         -------
@@ -364,6 +473,7 @@ def _get_debug_context(debug_ports, debug_args, debugger_path, container_env_var
             debug_ports=debug_ports,
             debug_args=debug_args,
             debugger_path=debugger_path,
+            debug_function=debug_function,
             container_env_vars=container_env_vars,
         )
 

samcli/commands/local/cli_common/options.py

@@ -6,6 +6,7 @@
 import click
 
 from samcli.commands._utils.options import template_click_option, docker_click_options, parameter_override_click_option
+from samcli.commands.local.cli_common.invoke_context import ContainersInitializationMode
 
 
 def get_application_dir():
@@ -140,3 +141,32 @@ def invoke_common_options(f):
         option(f)
 
     return f
+
+
+def warm_containers_common_options(f):
+    """
+    Warm containers related CLI options shared by "local start-api" and "local start_lambda" commands
+
+    :param f: Callback passed by Click
+    """
+
+    warm_containers_options = [
+        click.option(
+            "--warm-containers",
+            help="Optional. Specifies how AWS SAM CLI manages containers for each function.",
+            type=click.Choice(ContainersInitializationMode.__members__, case_sensitive=False),
+        ),
+        click.option(
+            "--debug-function",
+            help="Optional. Specifies the Lambda Function logicalId to apply debug options to when"
+            " --warm-containers is specified ",
+            type=click.STRING,
+            multiple=False,
+        ),
+    ]
+
+    # Reverse the list to maintain ordering of options in help text printed with --help
+    for option in reversed(warm_containers_options):
+        option(f)
+
+    return f

samcli/commands/local/lib/debug_context.py

@@ -4,19 +4,24 @@
 
 
 class DebugContext:
-    def __init__(self, debug_ports=None, debugger_path=None, debug_args=None, container_env_vars=None):
+    def __init__(
+        self, debug_ports=None, debugger_path=None, debug_args=None, debug_function=None, container_env_vars=None
+    ):
         """
         Initialize the Debug Context with Lambda debugger options
 
         :param tuple(int) debug_ports: Collection of debugger ports to be exposed from a docker container
         :param Path debugger_path: Path to a debugger to be launched
         :param string debug_args: Additional arguments to be passed to the debugger
+        :param string debug_function: The Lambda function logicalId that will have the debugging options enabled in case
+        of warm containers option is enabled
         :param dict container_env_vars: Additional environmental variables to be set.
         """
 
         self.debug_ports = debug_ports
         self.debugger_path = debugger_path
         self.debug_args = debug_args
+        self.debug_function = debug_function
         self.container_env_vars = container_env_vars
 
     def __bool__(self):

samcli/commands/local/lib/local_lambda.py

@@ -104,7 +104,7 @@ def invoke(self, function_name, event, stdout=None, stderr=None):
                     f"ImageUri not provided for Function: {function_name} of PackageType: {function.packagetype}"
                 )
             LOG.info("Invoking Container created from %s", function.imageuri)
-        config = self._get_invoke_config(function)
+        config = self.get_invoke_config(function)
 
         # Invoke the function
         try:
@@ -135,7 +135,7 @@ def is_debugging(self):
         """
         return bool(self.debug_context)
 
-    def _get_invoke_config(self, function):
+    def get_invoke_config(self, function):
         """
         Returns invoke configuration to pass to Lambda Runtime to invoke the given function