feat: Add support to build automatically npm dependencies

README.md

@@ -403,6 +403,11 @@ source_path = [
       "!vendor/colorful-.+.dist-info/.*",
       "!vendor/colorful/__pycache__/?.*",
     ]
+  }, {
+    path             = "src/nodejs14.x-app1",
+    npm_requirements = true,
+    npm_tmp_dir      = "/tmp/dir/location"
+    prefix_in_zip    = "foo/bar1",
   }, {
     path     = "src/python3.8-app3",
     commands = [
@@ -424,8 +429,9 @@ source_path = [
 ]
 ```
 
-Few notes:
+*Few notes:*
 
+- If you specify a source path as a string that references a folder and the runtime is either python or nodejs, the build process will automatically build python and nodejs dependencies if `requirements.txt` or `package.json` file will be found in the source folder. If you want to customize this behavior, please use the object notation as explained below.
 - All arguments except `path` are optional.
 - `patterns` - List of Python regex filenames should satisfy. Default value is "include everything" which is equal to `patterns = [".*"]`. This can also be specified as multiline heredoc string (no comments allowed). Some examples of valid patterns:
 
@@ -442,10 +448,12 @@ Few notes:
     !abc/def/hgk/.*    # Filter out again in abc/def/hgk sub folder
 ```
 
-- `commands` - List of commands to run. If specified, this argument overrides `pip_requirements`.
+- `commands` - List of commands to run. If specified, this argument overrides `pip_requirements` and `npm_requirements`.
   - `:zip [source] [destination]` is a special command which creates content of current working directory (first argument) and places it inside of path (second argument).
 - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead.
 - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds.
+- `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead.
+- `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds.
 - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive.
 
 ### Building in Docker

examples/build-package/README.md

@@ -45,6 +45,9 @@ Note that this example may create resources which cost money. Run `terraform des
 | <a name="module_package_with_docker"></a> [package\_with\_docker](#module\_package\_with\_docker) | ../../ | n/a |
 | <a name="module_package_with_patterns"></a> [package\_with\_patterns](#module\_package\_with\_patterns) | ../../ | n/a |
 | <a name="module_package_with_pip_requirements_in_docker"></a> [package\_with\_pip\_requirements\_in\_docker](#module\_package\_with\_pip\_requirements\_in\_docker) | ../../ | n/a |
+| <a name="module_package_dir_with_npm_install"></a> [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a |
+| <a name="module_package_dir_without_npm_install"></a> [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a |
+| <a name="module_package_with_npm_requirements_in_docker"></a> [package\_with\_npm\_requirements\_in_docker](#module\_package\_with\_npm\_requirements\_in_docker) | ../../ | n/a |
 
 ## Resources
 

examples/build-package/main.tf

@@ -223,6 +223,44 @@ module "package_with_docker" {
   docker_image      = "lambci/lambda:build-python3.8"
 }
 
+# Create zip-archive of a single directory where "npm install" will also be executed (default for nodejs runtime)
+module "package_dir_with_npm_install" {
+  source = "../../"
+
+  create_function = false
+
+  runtime     = "nodejs14.x"
+  source_path = "${path.module}/../fixtures/nodejs14.x-app1"
+}
+
+# Create zip-archive of a single directory without running "pip install" (which is default for python runtime)
+module "package_dir_without_npm_install" {
+  source = "../../"
+
+  create_function = false
+
+  runtime = "nodejs14.x"
+  source_path = [
+    {
+      path             = "${path.module}/../fixtures/nodejs14.x-app1"
+      npm_requirements = false
+      # npm_requirements = true  # Will run "npm install" with default requirements.txt
+    }
+  ]
+}
+
+# Create zip-archive of a single directory where "npm install" will also be executed using docker
+module "package_with_npm_requirements_in_docker" {
+  source = "../../"
+
+  create_function = false
+
+  runtime         = "nodejs14.x"
+  source_path     = "${path.module}/../fixtures/nodejs14.x-app1"
+  build_in_docker = true
+  hash_extra      = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install"
+}
+
 ################################
 # Build package in Docker and
 # use it to deploy Lambda Layer

examples/fixtures/nodejs14.x-app1/index.js

@@ -0,0 +1,16 @@
+'use strict';
+
+module.exports.hello = async (event) => {
+  console.log(event);
+  return {
+    statusCode: 200,
+    body: JSON.stringify(
+      {
+        message: `Go Serverless v3.0! Your Nodejs function executed successfully!`,
+        input: event,
+      },
+      null,
+      2
+    ),
+  };
+};

examples/fixtures/nodejs14.x-app1/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "nodejs14.x-app1",
+  "version": "1.0.0",
+  "main": "index.js",
+  "dependencies": {
+    "requests": "^0.3.0"
+  }
+}

package.py

@@ -660,6 +660,18 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None):
                 step('pip', runtime, requirements, prefix, tmp_dir)
                 hash(requirements)
 
+        def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None):
+            requirements = path
+            if os.path.isdir(path):
+                requirements = os.path.join(path, 'package.json')
+            if not os.path.isfile(requirements):
+                if required:
+                    raise RuntimeError(
+                        'File not found: {}'.format(requirements))
+            else:
+                step('npm', runtime, requirements, prefix, tmp_dir)
+                hash(requirements)
+
         def commands_step(path, commands):
             if not commands:
                 return
@@ -717,6 +729,9 @@ def commands_step(path, commands):
                 if runtime.startswith('python'):
                     pip_requirements_step(
                         os.path.join(path, 'requirements.txt'))
+                elif runtime.startswith('nodejs'):
+                    npm_requirements_step(
+                        os.path.join(path, 'package.json'))
                 step('zip', path, None)
                 hash(path)
 
@@ -731,6 +746,7 @@ def commands_step(path, commands):
                 else:
                     prefix = claim.get('prefix_in_zip')
                     pip_requirements = claim.get('pip_requirements')
+                    npm_requirements = claim.get('npm_package_json')
                     runtime = claim.get('runtime', query.runtime)
 
                     if pip_requirements and runtime.startswith('python'):
@@ -740,6 +756,13 @@ def commands_step(path, commands):
                             pip_requirements_step(pip_requirements, prefix,
                                                   required=True, tmp_dir=claim.get('pip_tmp_dir'))
 
+                    if npm_requirements and runtime.startswith('nodejs'):
+                        if isinstance(npm_requirements, bool) and path:
+                            npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir'))
+                        else:
+                            npm_requirements_step(npm_requirements, prefix,
+                                                  required=True, tmp_dir=claim.get('npm_tmp_dir'))
+
                     if path:
                         step('zip', path, prefix)
                         if patterns:
@@ -793,6 +816,16 @@ def execute(self, build_plan, zip_stream, query):
                         else:
                             # XXX: timestamp=0 - what actually do with it?
                             zs.write_dirs(rd, prefix=prefix, timestamp=0)
+            elif cmd == 'npm':
+                runtime, npm_requirements, prefix, tmp_dir = action[1:]
+                with install_npm_requirements(query, npm_requirements, tmp_dir) as rd:
+                    if rd:
+                        if pf:
+                            self._zip_write_with_filter(zs, pf, rd, prefix,
+                                                        timestamp=0)
+                        else:
+                            # XXX: timestamp=0 - what actually do with it?
+                            zs.write_dirs(rd, prefix=prefix, timestamp=0)
             elif cmd == 'sh':
                 r, w = os.pipe()
                 side_ch = os.fdopen(r)
@@ -934,6 +967,89 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
             yield temp_dir
 
 
+@contextmanager
+def install_npm_requirements(query, requirements_file, tmp_dir):
+    # TODO:
+    #  1. Emit files instead of temp_dir
+
+    if not os.path.exists(requirements_file):
+        yield
+        return
+
+    runtime = query.runtime
+    artifacts_dir = query.artifacts_dir
+    temp_dir = query.temp_dir
+    docker = query.docker
+    docker_image_tag_id = None
+
+    if docker:
+        docker_file = docker.docker_file
+        docker_image = docker.docker_image
+        docker_build_root = docker.docker_build_root
+
+        if docker_image:
+            ok = False
+            while True:
+                output = check_output(docker_image_id_command(docker_image))
+                if output:
+                    docker_image_tag_id = output.decode().strip()
+                    log.debug("DOCKER TAG ID: %s -> %s",
+                              docker_image, docker_image_tag_id)
+                    ok = True
+                if ok:
+                    break
+                docker_cmd = docker_build_command(
+                    build_root=docker_build_root,
+                    docker_file=docker_file,
+                    tag=docker_image,
+                )
+                check_call(docker_cmd)
+                ok = True
+        elif docker_file or docker_build_root:
+            raise ValueError('docker_image must be specified '
+                             'for a custom image future references')
+
+    log.info('Installing npm requirements: %s', requirements_file)
+    with tempdir(tmp_dir) as temp_dir:
+        requirements_filename = os.path.basename(requirements_file)
+        target_file = os.path.join(temp_dir, requirements_filename)
+        shutil.copyfile(requirements_file, target_file)
+
+        subproc_env = None
+        if not docker and OSX:
+            subproc_env = os.environ.copy()
+
+        # Install dependencies into the temporary directory.
+        with cd(temp_dir):
+            npm_command = ['npm', 'install']
+            if docker:
+                with_ssh_agent = docker.with_ssh_agent
+                chown_mask = '{}:{}'.format(os.getuid(), os.getgid())
+                shell_command = [shlex_join(npm_command), '&&',
+                                    shlex_join(['chown', '-R',
+                                                chown_mask, '.'])]
+                shell_command = [' '.join(shell_command)]
+                check_call(docker_run_command(
+                    '.', shell_command, runtime,
+                    image=docker_image_tag_id,
+                    shell=True, ssh_agent=with_ssh_agent
+                ))
+            else:
+                cmd_log.info(shlex_join(npm_command))
+                log_handler and log_handler.flush()
+                try:
+                    check_call(npm_command, env=subproc_env)
+                except FileNotFoundError as e:
+                    raise RuntimeError(
+                        "Nodejs interpreter version equal "
+                        "to defined lambda runtime ({}) should be "
+                        "available in system PATH".format(runtime)
+                    ) from e
+
+            os.remove(target_file)
+            yield temp_dir
+
+
 def docker_image_id_command(tag):
     """"""
     docker_cmd = ['docker', 'images', '--format={{.ID}}', tag]
@@ -1011,7 +1127,7 @@ def docker_run_command(build_root, command, runtime,
             ])
 
     if not image:
-        image = 'lambci/lambda:build-{}'.format(runtime)
+        image = 'public.ecr.aws/sam/build-{}'.format(runtime)
 
     docker_cmd.append(image)
 
@@ -1128,7 +1244,7 @@ def prepare_command(args):
 def build_command(args):
     """
     Builds a zip file from the source_dir or source_file.
-    Installs dependencies with pip automatically.
+    Installs dependencies with pip or npm automatically.
     """
 
     log = logging.getLogger('build')