diff --git a/.appveyor.yml b/.appveyor.yml index 94bf1e486..67aa9f515 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,7 +6,7 @@ image: environment: GOVERSION: 1.11 GRADLE_OPTS: -Dorg.gradle.daemon=false - nodejs_version: "10.10.0" + nodejs_version: "14.17.6" matrix: - PYTHON: "C:\\Python36-x64" @@ -93,7 +93,7 @@ for: - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" - sh: "rvm use 2.5" - sh: "nvm install ${nodejs_version}" - - sh: "npm install npm@5.6.0 -g" + - sh: "npm install npm@7.24.2 -g" - sh: "npm -v" - sh: "echo $PATH" - sh: "java --version" diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index f6524c659..bcacb67f7 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -69,7 +69,9 @@ def build( dependencies_dir=None, combine_dependencies=True, architecture=X86_64, + experimental_flags=None, ): + # pylint: disable-msg=too-many-locals """ Actually build the code by running workflows @@ -127,6 +129,10 @@ def build( :type architecture: str :param architecture: Type of architecture x86_64 and arm64 for Lambda Function + + :type experimental_flags: list + :param experimental_flags: + List of strings, which will indicate enabled experimental flags for the current build session """ if not os.path.exists(scratch_dir): @@ -146,6 +152,7 @@ def build( dependencies_dir=dependencies_dir, combine_dependencies=combine_dependencies, architecture=architecture, + experimental_flags=experimental_flags, ) return workflow.run() diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index 7ff19534c..e979a8392 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -164,6 +164,7 @@ def __init__( dependencies_dir=None, combine_dependencies=True, architecture=X86_64, + experimental_flags=None, ): """ Initialize the builder with given arguments. These arguments together form the "public API" that each @@ -200,6 +201,8 @@ def __init__( from dependency_folder into build folder architecture : str, optional Architecture type either arm64 or x86_64 for which the build will be based on in AWS lambda, by default X86_64 + experimental_flags: list, optional + List of strings, which will indicate enabled experimental flags for the current build session """ self.source_dir = source_dir @@ -215,6 +218,7 @@ def __init__( self.dependencies_dir = dependencies_dir self.combine_dependencies = combine_dependencies self.architecture = architecture + self.experimental_flags = experimental_flags if experimental_flags else [] # Actions are registered by the subclasses as they seem fit self.actions = [] diff --git a/aws_lambda_builders/workflows/nodejs_npm/DESIGN.md b/aws_lambda_builders/workflows/nodejs_npm/DESIGN.md index 803052ed7..b947573a9 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/DESIGN.md +++ b/aws_lambda_builders/workflows/nodejs_npm/DESIGN.md @@ -2,9 +2,7 @@ ### Scope -This package is an effort to port the Claudia.JS packager to a library that can -be used to handle the dependency resolution portion of packaging NodeJS code -for use in AWS Lambda. The scope for this builder is to take an existing +The scope for this builder is to take an existing directory containing customer code, including a valid `package.json` manifest specifying third-party dependencies. The builder will use NPM to include production dependencies and exclude test resources in a way that makes them @@ -24,9 +22,16 @@ To speed up Lambda startup time and optimise usage costs, the correct thing to do in most cases is just to package up production dependencies. During development work we can expect that the local `node_modules` directory contains all the various dependency types, and NPM does not provide a way to directly identify -just the ones relevant for production. To identify production dependencies, -this packager needs to copy the source to a clean temporary directory and re-run -dependency installation there. +just the ones relevant for production. + +There are two ways to include only production dependencies in a package: + +1. **without a bundler**: Copy the source to a clean temporary directory and + re-run dependency installation there. + +2. **with a bundler**: Apply a javascript code bundler (such as `esbuild` or + `webpack`) to produce a single-file javascript bundle by recursively + resolving included dependencies, starting from the main lambda handler. A frequently used trick to speed up NodeJS Lambda deployment is to avoid bundling the `aws-sdk`, since it is already available on the Lambda VM. @@ -53,7 +58,9 @@ far from optimal to create a stand-alone module. Copying would lead to significa larger packages than necessary, as sub-modules might still have test resources, and common references from multiple projects would be duplicated. -NPM also uses a locking mechanism (`package-lock.json`) that's in many ways more +NPM also uses two locking mechanisms (`package-lock.json` and `npm-shrinkwrap.json`) +that can be used to freeze versions of dependencies recursively, and provide reproducible +builds. Before version 7, the locking mechanism was in many ways more broken than functional, as it in some cases hard-codes locks to local disk paths, and gets confused by including the same package as a dependency throughout the project tree in different dependency categories @@ -73,10 +80,96 @@ To fully deal with those cases, this packager may need to execute the dependency installation step on a Docker image compatible with the target Lambda environment. -### Implementation +### Choosing the packaging type + +For a large majority of projects, packaging using a bundler has significant +advantages (speed and runtime package size, supporting local dependencies). + +However, there are also some drawbacks to using a bundler for a small set of +use cases (namely including packages with binary dependencies, such as `sharp`, a +popular image processing library). + +Because of this, it's important to support both ways of packaging. The version +without a bundler is slower, but will be correct in case of binary dependencies. +For backwards compatibility, this should be the default. + +Users should be able to activate packaging with a bundler for projects where that +is safe to do, such as those without any binary dependencies. + +The proposed approach is to use a "aws-sam" property in the package manifest +(`package.json`). If the `nodejs_npm` Lambda builder finds a matching property, it +knows that it is safe to use the bundler to package. + +The rest of this section outlines the major differences between packaging with +and without a bundler. + +#### packaging speed + +Packaging without a bundler is slower than using a bundler, as it +requires copying the project to a clean working directory, installing +dependencies and archiving into a single ZIP. + +Packaging with a bundler runs directly on files already on the disk, without +the need to copy or move files around. This approach can use the fast `npm ci` +command to just ensure that the dependencies are present on the disk instead of +always downloading all the dependencies. + +#### additional tools + +Packaging without a bundler does not require additional tools installed on the +development environment or CI systems, as it can just work with NPM. + +Packaging with a bundler requires installing additional tools (eg `esbuild`). + +#### handling local dependencies + +Packaging without a bundler requires complex +rewriting to handle local dependencies, and recursively packaging archives. In +theory, this was going to be implemented as a subsequent release after the +initial version of the `npm_nodejs` builder, but due to issues with container +environments and how `aws-lambda-builders` mounts the working directory, it was +not added for several years, and likely will not be implemented soon. + +Packaging with a bundler can handle local dependencies out of the box, since +it just traverses relative file liks. + +#### including non-javascript files + +Packaging without a bundler zips up entire contents of NPM packages. + +Packaging with a bundler only locates JavaScript files in the dependency tree. + +Some NPM packages include important binaries or resources in the NPM package, +which would not be included in the package without a bundler. This means that +packaging using a bundler is not universally applicable, and may never fully +replace packaging without a bundler. + +Some NPM packages include a lot of additional files not required at runtime. +`aws-sdk` for JavaScript (v2) is a good example, including TypeScript type +definitions, documentation and REST service definitions for automated code +generators. Packaging without a bundler includes these files as well, +unnecessarily increasing Lambda archive size. Packaging with a bundler just +ignores all these additional files out of the box. + +#### error reporting + +Packaging without a bundler leaves original file names and line numbers, ensuring +that any stack traces or exception reports correspond directly to the original +source files. + +Packaging with a bundler creates a single file from all the dependencies, so +stack traces on production no longer correspond to original source files. As a +workaround, bundlers can include a 'source map' file, to allow translating +production stack traces into source stack traces. Prior to Node 14, this +required including a separate NPM package, or additional tools. Since Node 14, +stack trace translation can be [activated using an environment +variable](https://serverless.pub/aws-lambda-node-sourcemaps/) + + +### Implementation without a bundler The general algorithm for preparing a node package for use on AWS Lambda -is as follows. +without a JavaScript bundler (`esbuild` or `webpack`) is as follows. #### Step 1: Prepare a clean copy of the project source files @@ -134,3 +227,157 @@ To fully support dependencies that download or compile binaries for a target pla needs to be executed inside a Docker image compatible with AWS Lambda. _(out of scope for the current version)_ +### Implementation with a bundler + +The general algorithm for preparing a node package for use on AWS Lambda +with a bundler (`esbuild` or `webpack`) is as follows. + +#### Step 1: ensure production dependencies are installed + +If the directory contains `package-lock.json` or `npm-shrinkwrap.json`, +execute [`npm ci`](https://docs.npmjs.com/cli/v7/commands/npm-ci). This +operation is designed to be faster than installing dependencies using `npm install` +in automated CI environments. + +If the directory does not contain lockfiles, but contains `package.json`, +execute [`npm install --production`] to download production dependencies. + +#### Step 2: bundle the main Lambda file + +Execute `esbuild` to produce a single JavaScript file by recursively resolving +included dependencies, and optionally a source map. + +Ensure that the target file name is the same as the entry point of the Lambda +function, so that there is no impact on the CloudFormation template. + + +### Activating the bundler workflow + +Because there are advantages and disadvantages to both approaches (with and +without a bundler), the user should be able to choose between them. The default +is not to use a bundler (both because it's universally applicable and for +backwards compatibility). Node.js pakage manifests (`package.json`) allow for +custom properties, so a user can activate the bundler process by providing an +`aws_sam` configuration property in the package manifest. If this property is +present in the package manifest, and the sub-property `bundler` equals +`esbuild`, the Node.js NPM Lambda builder activates the bundler process. + +Because the Lambda builder workflow is not aware of the main lambda function +definition, (the file containing the Lambda handler function) the user must +also specify the main entry point for bundling . This is a bit of an +unfortunate duplication with SAM Cloudformation template, but with the current +workflow design there is no way around it. + +In addition, as a single JavaScript source package can contain multiple functions, +and can be included multiple times in a single CloudFormation template, it's possible +that there may be multiple entry points for bundling. SAM build executes the build +only once for the function in this case, so all entry points have to be bundled +at once. + +The following example is a minimal `package.json` to activate the `esbuild` bundler +on a javascript file, starting from `lambda.js`. It will produce a bundled `lambda.js` +in the artifacts folder. + +```json +{ + "name": "nodeps-esbuild", + "version": "1.0.0", + "license": "APACHE2.0", + "aws_sam": { + "bundler": "esbuild", + "entry_points": ["lambda.js"] + } +} +``` + +#### Locating the esbuild binary + +`esbuild` supports platform-independent binary distribution using NPM, by +including the `esbuild` package as a dependency. The Lambda builder should +first try to locate the binary in the Lambda code repository (allowing the +user to include a specific version). Failing that, the Lambda builder should +try to locate the `esbuild` binary in the `executable_search_paths` configured +for the workflow, then the operating system `PATH` environment variable. + +The Lambda builder **should not** bring its own `esbuild` binary, but it should +clearly point to the error when one is not found, to allow users to configure the +build correctly. + +In the previous example, the esbuild binary is not included in the package dependencies, +so the Lambda builder will use the system executable paths to search for it. In the +example below, `esbuild` is included in the package, so the Lambda builder should use it +directly. + +```json +{ + "name": "with-deps-esbuild", + "version": "1.0.0", + "license": "APACHE2.0", + "aws_sam": { + "bundler": "esbuild", + "entry_points": ["lambda.js"] + }, + "devDependencies": { + "esbuild": "^0.11.23" + } +} +``` + +For a full example, see the [`with-deps-esbuild`](../../../tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/) test project. + +#### Building typescript + +`esbuild` supports bundling typescript out of the box and transpiling it to plain +javascript. The user just needs to point to a typescript file as the main entry point, +as in the example below. There is no transpiling process needed upfront. + + +```js +{ + "name": "with-deps-esbuild-typescript", + "version": "1.0.0", + "license": "APACHE2.0", + "aws_sam": { + "bundler": "esbuild", + "entry_points": ["included.ts"] + }, + "dependencies": { + "@types/aws-lambda": "^8.10.76" + }, + "devDependencies": { + "esbuild": "^0.11.23" + } +} +``` + +For a full example, see the [`with-deps-esbuild-typescript`](../../../tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/) test project. + +**important note:** esbuild does not perform type checking, so users wanting to ensure type-checks need to run the `tsc` process as part of their +testing flow before invoking `sam build`. For additional typescript caveats with esbuild, check out . + +#### Configuring the bundler + +The Lambda builder invokes `esbuild` with sensible defaults that will work for the majority of cases. Importantly, the following three parameters are set by default + +* `--minify`, as it [produces a smaller runtime package](https://esbuild.github.io/api/#minify) +* `--sourcemap`, as it generates a [source map that allows for correct stack trace reporting](https://esbuild.github.io/api/#sourcemap) in case of errors (see the [Error reporting](#error-reporting) section above) +* `--target es2020`, as it allows for javascript features present in Node 14 + +Users might want to tweak some of these runtime arguments for a specific project, for example not including the source map to further reduce the package size, or restricting javascript features to an older version. The Lambda builder allows this with optional sub-properties of the `aws_sam` configuration property. + +* `target`: string, corresponding to a supported [esbuild target](https://esbuild.github.io/api/#target) property +* `minify`: boolean, defaulting to `true` +* `sourcemap`: boolean, defaulting to `true` + +Here is an example that deactivates minification and source maps, and supports JavaScript features compatible with Node.js version 10. + +```json +{ + "aws_sam": { + "bundler": "esbuild", + "entry_points": ["included.ts"], + "target": "node10", + "minify": false, + "sourcemap": false + } +} diff --git a/aws_lambda_builders/workflows/nodejs_npm/actions.py b/aws_lambda_builders/workflows/nodejs_npm/actions.py index 43259bd8c..cb220d7f3 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/actions.py +++ b/aws_lambda_builders/workflows/nodejs_npm/actions.py @@ -6,6 +6,7 @@ from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError from .npm import NpmExecutionError +from .esbuild import EsbuildExecutionError LOG = logging.getLogger(__name__) @@ -80,7 +81,7 @@ class NodejsNpmInstallAction(BaseAction): DESCRIPTION = "Installing dependencies from NPM" PURPOSE = Purpose.RESOLVE_DEPENDENCIES - def __init__(self, artifacts_dir, subprocess_npm): + def __init__(self, artifacts_dir, subprocess_npm, is_production=True): """ :type artifacts_dir: str :param artifacts_dir: an existing (writable) directory with project source files. @@ -88,11 +89,15 @@ def __init__(self, artifacts_dir, subprocess_npm): :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm :param subprocess_npm: An instance of the NPM process wrapper + + :type is_production: bool + :param is_production: NPM installation mode is production (eg --production=false to force dev dependencies) """ super(NodejsNpmInstallAction, self).__init__() self.artifacts_dir = artifacts_dir self.subprocess_npm = subprocess_npm + self.is_production = is_production def execute(self): """ @@ -101,17 +106,62 @@ def execute(self): :raises lambda_builders.actions.ActionFailedError: when NPM execution fails """ + mode = "--production" if self.is_production else "--production=false" + try: LOG.debug("NODEJS installing in: %s", self.artifacts_dir) self.subprocess_npm.run( - ["install", "-q", "--no-audit", "--no-save", "--production", "--unsafe-perm"], cwd=self.artifacts_dir + ["install", "-q", "--no-audit", "--no-save", mode, "--unsafe-perm"], cwd=self.artifacts_dir ) except NpmExecutionError as ex: raise ActionFailedError(str(ex)) +class NodejsNpmCIAction(BaseAction): + + """ + A Lambda Builder Action that installs NPM project dependencies + using the CI method - which is faster and better reproducible + for CI environments, but requires a lockfile (package-lock.json + or npm-shrinkwrap.json) + """ + + NAME = "NpmCI" + DESCRIPTION = "Installing dependencies from NPM using the CI method" + PURPOSE = Purpose.RESOLVE_DEPENDENCIES + + def __init__(self, artifacts_dir, subprocess_npm): + """ + :type artifacts_dir: str + :param artifacts_dir: an existing (writable) directory with project source files. + Dependencies will be installed in this directory. + + :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm + :param subprocess_npm: An instance of the NPM process wrapper + """ + + super(NodejsNpmCIAction, self).__init__() + self.artifacts_dir = artifacts_dir + self.subprocess_npm = subprocess_npm + + def execute(self): + """ + Runs the action. + + :raises lambda_builders.actions.ActionFailedError: when NPM execution fails + """ + + try: + LOG.debug("NODEJS installing ci in: %s", self.artifacts_dir) + + self.subprocess_npm.run(["ci"], cwd=self.artifacts_dir) + + except NpmExecutionError as ex: + raise ActionFailedError(str(ex)) + + class NodejsNpmrcCopyAction(BaseAction): """ @@ -185,7 +235,7 @@ def execute(self): """ Runs the action. - :raises lambda_builders.actions.ActionFailedError: when .npmrc copying fails + :raises lambda_builders.actions.ActionFailedError: when deleting .npmrc fails """ try: @@ -196,3 +246,120 @@ def execute(self): except OSError as ex: raise ActionFailedError(str(ex)) + + +class NodejsNpmLockFileCleanUpAction(BaseAction): + + """ + A Lambda Builder Action that cleans up garbage lockfile left by 7 in node_modules + """ + + NAME = "LockfileCleanUp" + DESCRIPTION = "Cleans garbage lockfiles dir" + PURPOSE = Purpose.COPY_SOURCE + + def __init__(self, artifacts_dir, osutils): + """ + :type artifacts_dir: str + :param artifacts_dir: an existing (writable) directory with project source files. + Dependencies will be installed in this directory. + + :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils + :param osutils: An instance of OS Utilities for file manipulation + """ + + super(NodejsNpmLockFileCleanUpAction, self).__init__() + self.artifacts_dir = artifacts_dir + self.osutils = osutils + + def execute(self): + """ + Runs the action. + + :raises lambda_builders.actions.ActionFailedError: when deleting the lockfile fails + """ + + try: + npmrc_path = self.osutils.joinpath(self.artifacts_dir, "node_modules", ".package-lock.json") + if self.osutils.file_exists(npmrc_path): + LOG.debug(".package-lock cleanup in: %s", self.artifacts_dir) + self.osutils.remove_file(npmrc_path) + + except OSError as ex: + raise ActionFailedError(str(ex)) + + +class EsbuildBundleAction(BaseAction): + + """ + A Lambda Builder Action that packages a Node.js package using esbuild into a single file + optionally transpiling TypeScript + """ + + NAME = "EsbuildBundle" + DESCRIPTION = "Packaging source using Esbuild" + PURPOSE = Purpose.COPY_SOURCE + + def __init__(self, source_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild): + """ + :type source_dir: str + :param source_dir: an existing (readable) directory containing source files + + + :type artifacts_dir: str + :param artifacts_dir: an existing (writable) directory where to store the output. + Note that the actual result will be in the 'package' subdirectory here. + + :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils + :param osutils: An instance of OS Utilities for file manipulation + + :type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessEsbuild + :param subprocess_esbuild: An instance of the Esbuild process wrapper + """ + super(EsbuildBundleAction, self).__init__() + self.source_dir = source_dir + self.artifacts_dir = artifacts_dir + self.bundler_config = bundler_config + self.osutils = osutils + self.subprocess_esbuild = subprocess_esbuild + + def execute(self): + """ + Runs the action. + + :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails + """ + + if "entry_points" not in self.bundler_config: + raise ActionFailedError("entry_points not set ({})".format(self.bundler_config)) + + entry_points = self.bundler_config["entry_points"] + + if not isinstance(entry_points, list): + raise ActionFailedError("entry_points must be a list ({})".format(self.bundler_config)) + + if not entry_points: + raise ActionFailedError("entry_points must not be empty ({})".format(self.bundler_config)) + + entry_paths = [self.osutils.joinpath(self.source_dir, entry_point) for entry_point in entry_points] + + LOG.debug("NODEJS building %s using esbuild to %s", entry_paths, self.artifacts_dir) + + for entry_point in entry_paths: + if not self.osutils.file_exists(entry_point): + raise ActionFailedError("entry point {} does not exist".format(entry_point)) + + args = entry_points + ["--bundle", "--platform=node", "--format=cjs"] + minify = self.bundler_config.get("minify", True) + sourcemap = self.bundler_config.get("sourcemap", True) + target = self.bundler_config.get("target", "es2020") + if minify: + args.append("--minify") + if sourcemap: + args.append("--sourcemap") + args.append("--target={}".format(target)) + args.append("--outdir={}".format(self.artifacts_dir)) + try: + self.subprocess_esbuild.run(args, cwd=self.source_dir) + except EsbuildExecutionError as ex: + raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/nodejs_npm/esbuild.py b/aws_lambda_builders/workflows/nodejs_npm/esbuild.py new file mode 100644 index 000000000..992c49fe5 --- /dev/null +++ b/aws_lambda_builders/workflows/nodejs_npm/esbuild.py @@ -0,0 +1,105 @@ +""" +Wrapper around calling esbuild through a subprocess. +""" + +import logging + +LOG = logging.getLogger(__name__) + + +class EsbuildExecutionError(Exception): + + """ + Exception raised in case NPM execution fails. + It will pass on the standard error output from the NPM console. + """ + + MESSAGE = "Esbuild Failed: {message}" + + def __init__(self, **kwargs): + Exception.__init__(self, self.MESSAGE.format(**kwargs)) + + +class SubprocessEsbuild(object): + + """ + Wrapper around the Esbuild command line utility, making it + easy to consume execution results. + """ + + def __init__(self, osutils, executable_search_paths, which): + """ + :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils + :param osutils: An instance of OS Utilities for file manipulation + + :type executable_search_paths: list + :param executable_search_paths: List of paths to the NPM package binary utilities. This will + be used to find embedded esbuild at runtime if present in the package + + :type which: aws_lambda_builders.utils.which + :param which: Function to get paths which conform to the given mode on the PATH + with the prepended additional search paths + """ + self.osutils = osutils + self.executable_search_paths = executable_search_paths + self.which = which + + def esbuild_binary(self): + """ + Finds the esbuild binary at runtime. + + The utility may be present as a package dependency of the Lambda project, + or in the global path. If there is one in the Lambda project, it should + be preferred over a global utility. The check has to be executed + at runtime, since NPM dependencies will be installed by the workflow + using one of the previous actions. + """ + + LOG.debug("checking for esbuild in: %s", self.executable_search_paths) + binaries = self.which("esbuild", executable_search_paths=self.executable_search_paths) + LOG.debug("potential esbuild binaries: %s", binaries) + + if binaries: + return binaries[0] + else: + raise EsbuildExecutionError(message="cannot find esbuild") + + def run(self, args, cwd=None): + + """ + Runs the action. + + :type args: list + :param args: Command line arguments to pass to Esbuild + + :type cwd: str + :param cwd: Directory where to execute the command (defaults to current dir) + + :rtype: str + :return: text of the standard output from the command + + :raises aws_lambda_builders.workflows.nodejs_npm.npm.EsbuildExecutionError: + when the command executes with a non-zero return code. The exception will + contain the text of the standard error output from the command. + + :raises ValueError: if arguments are not provided, or not a list + """ + + if not isinstance(args, list): + raise ValueError("args must be a list") + + if not args: + raise ValueError("requires at least one arg") + + invoke_esbuild = [self.esbuild_binary()] + args + + LOG.debug("executing Esbuild: %s", invoke_esbuild) + + p = self.osutils.popen(invoke_esbuild, stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=cwd) + + out, err = p.communicate() + + if p.returncode != 0: + raise EsbuildExecutionError(message=err.decode("utf8").strip()) + + return out.decode("utf8").strip() diff --git a/aws_lambda_builders/workflows/nodejs_npm/utils.py b/aws_lambda_builders/workflows/nodejs_npm/utils.py index ad92cfd23..dc114ba06 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/utils.py +++ b/aws_lambda_builders/workflows/nodejs_npm/utils.py @@ -7,6 +7,9 @@ import tarfile import subprocess import shutil +import json + +EXPERIMENTAL_FLAG_ESBUILD = "experimentalEsbuild" class OSUtils(object): @@ -48,3 +51,14 @@ def abspath(self, path): def is_windows(self): return platform.system().lower() == "windows" + + def parse_json(self, path): + with open(path) as json_file: + return json.load(json_file) + + +def is_experimental_esbuild_scope(experimental_flags): + """ + A function which will determine if experimental esbuild scope is active + """ + return bool(experimental_flags) and EXPERIMENTAL_FLAG_ESBUILD in experimental_flags diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index b890ddcf1..b8f209c20 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -1,14 +1,34 @@ """ NodeJS NPM Workflow """ + import logging +import json +from typing import List from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.workflow import BaseWorkflow, Capability -from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction, MoveDependenciesAction, CleanUpAction -from .actions import NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction -from .utils import OSUtils +from aws_lambda_builders.actions import ( + CopySourceAction, + CleanUpAction, + CopyDependenciesAction, + MoveDependenciesAction, + BaseAction, +) +from aws_lambda_builders.utils import which +from aws_lambda_builders.exceptions import WorkflowFailedError +from .actions import ( + NodejsNpmPackAction, + NodejsNpmLockFileCleanUpAction, + NodejsNpmInstallAction, + NodejsNpmrcCopyAction, + NodejsNpmrcCleanUpAction, + NodejsNpmCIAction, + EsbuildBundleAction, +) +from .utils import OSUtils, is_experimental_esbuild_scope from .npm import SubprocessNpm +from .esbuild import SubprocessEsbuild LOG = logging.getLogger(__name__) @@ -26,6 +46,8 @@ class NodejsNpmWorkflow(BaseWorkflow): EXCLUDED_FILES = (".aws-sam", ".git") + CONFIG_PROPERTY = "aws_sam" + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): super(NodejsNpmWorkflow, self).__init__( @@ -35,23 +57,57 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim if osutils is None: osutils = OSUtils() - subprocess_npm = SubprocessNpm(osutils) - - tar_dest_dir = osutils.joinpath(scratch_dir, "unpacked") - tar_package_dir = osutils.joinpath(tar_dest_dir, "package") - if not osutils.file_exists(manifest_path): LOG.warning("package.json file not found. Continuing the build without dependencies.") self.actions = [CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)] return + subprocess_npm = SubprocessNpm(osutils) + + manifest_config = self.get_manifest_config(osutils, manifest_path) + + if manifest_config["bundler"] == "esbuild" and is_experimental_esbuild_scope(self.experimental_flags): + self.actions = self.actions_with_bundler( + source_dir, artifacts_dir, manifest_config, osutils, subprocess_npm + ) + else: + self.actions = self.actions_without_bundler( + source_dir, artifacts_dir, scratch_dir, manifest_path, osutils, subprocess_npm + ) + + def actions_without_bundler(self, source_dir, artifacts_dir, scratch_dir, manifest_path, osutils, subprocess_npm): + """ + Generate a list of Nodejs build actions without a bundler + + :type source_dir: str + :param source_dir: an existing (readable) directory containing source files + + :type artifacts_dir: str + :param artifacts_dir: an existing (writable) directory where to store the output. + + :type scratch_dir: str + :param scratch_dir: an existing (writable) directory for temporary files + + :type manifest_path: str + :param manifest_path: path to package.json of an NPM project with the source to pack + + :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils + :param osutils: An instance of OS Utilities for file manipulation + + :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm + :param subprocess_npm: An instance of the NPM process wrapper + + :rtype: list + :return: List of build actions to execute + """ + tar_dest_dir = osutils.joinpath(scratch_dir, "unpacked") + tar_package_dir = osutils.joinpath(tar_dest_dir, "package") npm_pack = NodejsNpmPackAction( tar_dest_dir, scratch_dir, manifest_path, osutils=osutils, subprocess_npm=subprocess_npm ) - npm_copy_npmrc = NodejsNpmrcCopyAction(tar_package_dir, source_dir, osutils=osutils) - self.actions = [ + actions = [ npm_pack, npm_copy_npmrc, CopySourceAction(tar_package_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), @@ -59,31 +115,98 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim if self.download_dependencies: # installed the dependencies into artifact folder - self.actions.append(NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm)) + actions.append(NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm)) # if dependencies folder exists, copy or move dependencies from artifact folder to dependencies folder # depends on the combine_dependencies flag if self.dependencies_dir: # clean up the dependencies folder first - self.actions.append(CleanUpAction(self.dependencies_dir)) + actions.append(CleanUpAction(self.dependencies_dir)) # if combine_dependencies is set, we should keep dependencies and source code in the artifact folder # while copying the dependencies. Otherwise we should separate the dependencies and source code if self.combine_dependencies: - self.actions.append(CopyDependenciesAction(source_dir, artifacts_dir, self.dependencies_dir)) + actions.append(CopyDependenciesAction(source_dir, artifacts_dir, self.dependencies_dir)) else: - self.actions.append(MoveDependenciesAction(source_dir, artifacts_dir, self.dependencies_dir)) + actions.append(MoveDependenciesAction(source_dir, artifacts_dir, self.dependencies_dir)) else: # if dependencies folder exists and not download dependencies, simply copy the dependencies from the # dependencies folder to artifact folder if self.dependencies_dir and self.combine_dependencies: - self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) + actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) else: LOG.info( "download_dependencies is False and dependencies_dir is None. Copying the source files into the " "artifacts directory. " ) - self.actions.append(NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils)) + actions.append(NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils)) + actions.append(NodejsNpmLockFileCleanUpAction(artifacts_dir, osutils=osutils)) + + if self.dependencies_dir: + actions.append(NodejsNpmLockFileCleanUpAction(self.dependencies_dir, osutils=osutils)) + + return actions + + def actions_with_bundler(self, source_dir, artifacts_dir, bundler_config, osutils, subprocess_npm): + """ + Generate a list of Nodejs build actions with a bundler + + :type source_dir: str + :param source_dir: an existing (readable) directory containing source files + + :type artifacts_dir: str + :param artifacts_dir: an existing (writable) directory where to store the output. + + :type bundler_config: dict + :param bundler_config: configurations for the bundler action + + :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils + :param osutils: An instance of OS Utilities for file manipulation + + :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm + :param subprocess_npm: An instance of the NPM process wrapper + + :rtype: list + :return: List of build actions to execute + """ + lockfile_path = osutils.joinpath(source_dir, "package-lock.json") + shrinkwrap_path = osutils.joinpath(source_dir, "npm-shrinkwrap.json") + npm_bin_path = subprocess_npm.run(["bin"], cwd=source_dir) + executable_search_paths = [npm_bin_path] + if self.executable_search_paths is not None: + executable_search_paths = executable_search_paths + self.executable_search_paths + subprocess_esbuild = SubprocessEsbuild(osutils, executable_search_paths, which=which) + + if osutils.file_exists(lockfile_path) or osutils.file_exists(shrinkwrap_path): + install_action = NodejsNpmCIAction(source_dir, subprocess_npm=subprocess_npm) + else: + install_action = NodejsNpmInstallAction(source_dir, subprocess_npm=subprocess_npm, is_production=False) + + esbuild_action = EsbuildBundleAction(source_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild) + return [install_action, esbuild_action] + + def get_manifest_config(self, osutils, manifest_path): + """ + Get the aws_sam specific properties from the manifest, if they exist. + + :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils + :param osutils: An instance of OS Utilities for file manipulation + + :type manifest_path: str + :param manifest_path: Path to the manifest file + + :rtype: dict + :return: Dict with aws_sam specific bundler configs + """ + LOG.debug("NODEJS reading manifest from %s", manifest_path) + try: + manifest = osutils.parse_json(manifest_path) + if self.CONFIG_PROPERTY in manifest and isinstance(manifest[self.CONFIG_PROPERTY], dict): + return manifest[self.CONFIG_PROPERTY] + else: + return {"bundler": ""} + except (OSError, json.decoder.JSONDecodeError) as ex: + raise WorkflowFailedError(workflow_name=self.NAME, action_name="ParseManifest", reason=str(ex)) def get_resolvers(self): """ diff --git a/tests/functional/workflows/nodejs_npm/test_data/test.json b/tests/functional/workflows/nodejs_npm/test_data/test.json new file mode 100644 index 000000000..a6867ce02 --- /dev/null +++ b/tests/functional/workflows/nodejs_npm/test_data/test.json @@ -0,0 +1 @@ +{"a":1,"b":{"c":2}} diff --git a/tests/functional/workflows/nodejs_npm/test_utils.py b/tests/functional/workflows/nodejs_npm/test_utils.py index bd39e0ff3..a45d5585f 100644 --- a/tests/functional/workflows/nodejs_npm/test_utils.py +++ b/tests/functional/workflows/nodejs_npm/test_utils.py @@ -123,3 +123,10 @@ def test_popen_can_accept_cwd(self): self.assertEqual(p.returncode, 0) self.assertEqual(out.decode("utf8").strip(), os.path.abspath(testdata_dir)) + + def test_parse_json_reads_json_contents_into_memory(self): + + json_file = os.path.join(os.path.dirname(__file__), "test_data", "test.json") + json_contents = self.osutils.parse_json(json_file) + self.assertEqual(json_contents["a"], 1) + self.assertEqual(json_contents["b"]["c"], 2) diff --git a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py index dcd7397f8..02a85cf8a 100644 --- a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py +++ b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py @@ -1,5 +1,4 @@ import logging -import mock import os import shutil import tempfile @@ -149,7 +148,7 @@ def test_fails_if_package_json_is_broken(self): runtime=self.runtime, ) - self.assertIn("Unexpected end of JSON input", str(ctx.exception)) + self.assertIn("NodejsNpmBuilder:ParseManifest", str(ctx.exception)) def test_builds_project_with_remote_dependencies_without_download_dependencies_with_dependencies_dir(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps") diff --git a/tests/integration/workflows/nodejs_npm/test_nodejs_npm_with_esbuild.py b/tests/integration/workflows/nodejs_npm/test_nodejs_npm_with_esbuild.py new file mode 100644 index 000000000..0de36f41d --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/test_nodejs_npm_with_esbuild.py @@ -0,0 +1,115 @@ +import os +import shutil +import tempfile +from unittest import TestCase +from aws_lambda_builders.builder import LambdaBuilder +from aws_lambda_builders.workflows.nodejs_npm.npm import SubprocessNpm +from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils, EXPERIMENTAL_FLAG_ESBUILD + + +class TestNodejsNpmWorkflowWithEsbuild(TestCase): + """ + Verifies that `nodejs_npm` workflow works by building a Lambda using NPM + """ + + TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata") + + def setUp(self): + self.artifacts_dir = tempfile.mkdtemp() + self.scratch_dir = tempfile.mkdtemp() + + self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps-esbuild") + + self.builder = LambdaBuilder(language="nodejs", dependency_manager="npm", application_framework=None) + self.runtime = "nodejs14.x" + + def tearDown(self): + shutil.rmtree(self.artifacts_dir) + shutil.rmtree(self.scratch_dir) + + def test_invokes_old_builder_without_feature_flag(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild") + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + ) + + expected_files = {"included.js", "node_modules", "excluded.js", "package.json"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + def test_builds_javascript_project_with_dependencies(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild") + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) + + expected_files = {"included.js", "included.js.map"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + def test_builds_javascript_project_with_multiple_entrypoints(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild-multiple-entrypoints") + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) + + expected_files = {"included.js", "included.js.map", "included2.js", "included2.js.map"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + def test_builds_typescript_projects(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild-typescript") + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) + + expected_files = {"included.js", "included.js.map"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + def test_builds_with_external_esbuild(self): + osutils = OSUtils() + npm = SubprocessNpm(osutils) + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps-esbuild") + esbuild_dir = os.path.join(self.TEST_DATA_FOLDER, "esbuild-binary") + + npm.run(["ci"], cwd=esbuild_dir) + + binpath = npm.run(["bin"], cwd=esbuild_dir) + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + executable_search_paths=[binpath], + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) + + expected_files = {"included.js", "included.js.map"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) diff --git a/tests/integration/workflows/nodejs_npm/testdata/esbuild-binary/package-lock.json b/tests/integration/workflows/nodejs_npm/testdata/esbuild-binary/package-lock.json new file mode 100644 index 000000000..9ce0cc19a --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/esbuild-binary/package-lock.json @@ -0,0 +1,31 @@ +{ + "name": "esbuild-binary", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "esbuild": "^0.11.23" + } + }, + "node_modules/esbuild": { + "version": "0.11.23", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.23.tgz", + "integrity": "sha512-iaiZZ9vUF5wJV8ob1tl+5aJTrwDczlvGP0JoMmnpC2B0ppiMCu8n8gmy5ZTGl5bcG081XBVn+U+jP+mPFm5T5Q==", + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + } + } + }, + "dependencies": { + "esbuild": { + "version": "0.11.23", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.23.tgz", + "integrity": "sha512-iaiZZ9vUF5wJV8ob1tl+5aJTrwDczlvGP0JoMmnpC2B0ppiMCu8n8gmy5ZTGl5bcG081XBVn+U+jP+mPFm5T5Q==" + } + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/esbuild-binary/package.json b/tests/integration/workflows/nodejs_npm/testdata/esbuild-binary/package.json new file mode 100644 index 000000000..eb70cca6e --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/esbuild-binary/package.json @@ -0,0 +1,14 @@ +{ + "name": "esbuild-binary", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "esbuild": "^0.11.23" + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/.gitignore b/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/.gitignore new file mode 100644 index 000000000..d8b83df9c --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/.gitignore @@ -0,0 +1 @@ +package-lock.json diff --git a/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/excluded.js b/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/excluded.js new file mode 100644 index 000000000..8bf8be437 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/excluded.js @@ -0,0 +1,2 @@ +//excluded +const x = 1; diff --git a/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/included.js b/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/included.js new file mode 100644 index 000000000..17fcc2576 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/included.js @@ -0,0 +1,3 @@ +//included +const x = 1; +module.exports = x; diff --git a/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/package.json b/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/package.json new file mode 100644 index 000000000..997d05815 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/no-deps-esbuild/package.json @@ -0,0 +1,12 @@ +{ + "name": "nodeps-esbuild", + "version": "1.0.0", + "description": "", + "keywords": [], + "author": "", + "license": "APACHE2.0", + "aws_sam": { + "bundler": "esbuild", + "entry_points": ["included.js"] + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/.gitignore b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/.gitignore new file mode 100644 index 000000000..d8b83df9c --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/.gitignore @@ -0,0 +1 @@ +package-lock.json diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/excluded.js b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/excluded.js new file mode 100644 index 000000000..8bf8be437 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/excluded.js @@ -0,0 +1,2 @@ +//excluded +const x = 1; diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/included.js b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/included.js new file mode 100644 index 000000000..17fcc2576 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/included.js @@ -0,0 +1,3 @@ +//included +const x = 1; +module.exports = x; diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/included2.js b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/included2.js new file mode 100644 index 000000000..2f7ab0b1e --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/included2.js @@ -0,0 +1,3 @@ +//included2 +const y = 1; +module.exports = y; diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/package.json b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/package.json new file mode 100644 index 000000000..e85d55506 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-multiple-entrypoints/package.json @@ -0,0 +1,18 @@ +{ + "name": "with-deps-esbuild-multiple-entrypoints", + "version": "1.0.0", + "description": "", + "keywords": [], + "author": "", + "license": "APACHE2.0", + "aws_sam": { + "bundler": "esbuild", + "entry_points": ["included.js", "included2.js"] + }, + "dependencies": { + "minimal-request-promise": "^1.5.0" + }, + "devDependencies": { + "esbuild": "^0.11.23" + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/included.ts b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/included.ts new file mode 100644 index 000000000..82397888a --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/included.ts @@ -0,0 +1,9 @@ +import { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda"; + +export const lambdaHandler = async (event: APIGatewayProxyEvent): Promise => { + const queries = JSON.stringify(event.queryStringParameters); + return { + statusCode: 200, + body: "OK" + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/package-lock.json b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/package-lock.json new file mode 100644 index 000000000..7230f5534 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/package-lock.json @@ -0,0 +1,46 @@ +{ + "name": "with-deps-esbuild-typescript", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "version": "1.0.0", + "license": "APACHE2.0", + "dependencies": { + "@types/aws-lambda": "^8.10.76" + }, + "devDependencies": { + "esbuild": "^0.11.23" + } + }, + "node_modules/@types/aws-lambda": { + "version": "8.10.76", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.76.tgz", + "integrity": "sha512-lCTyeRm3NWqSwDnoji0z82Pl0tsOpr1p+33AiNeidgarloWXh3wdiVRUuxEa+sY9S5YLOYGz5X3N3Zvpibvm5w==" + }, + "node_modules/esbuild": { + "version": "0.11.23", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.23.tgz", + "integrity": "sha512-iaiZZ9vUF5wJV8ob1tl+5aJTrwDczlvGP0JoMmnpC2B0ppiMCu8n8gmy5ZTGl5bcG081XBVn+U+jP+mPFm5T5Q==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + } + } + }, + "dependencies": { + "@types/aws-lambda": { + "version": "8.10.76", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.76.tgz", + "integrity": "sha512-lCTyeRm3NWqSwDnoji0z82Pl0tsOpr1p+33AiNeidgarloWXh3wdiVRUuxEa+sY9S5YLOYGz5X3N3Zvpibvm5w==" + }, + "esbuild": { + "version": "0.11.23", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.23.tgz", + "integrity": "sha512-iaiZZ9vUF5wJV8ob1tl+5aJTrwDczlvGP0JoMmnpC2B0ppiMCu8n8gmy5ZTGl5bcG081XBVn+U+jP+mPFm5T5Q==", + "dev": true + } + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/package.json b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/package.json new file mode 100644 index 000000000..43827dd4c --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild-typescript/package.json @@ -0,0 +1,18 @@ +{ + "name": "with-deps-esbuild-typescript", + "version": "1.0.0", + "description": "", + "keywords": [], + "author": "", + "license": "APACHE2.0", + "aws_sam": { + "bundler": "esbuild", + "entry_points": ["included.ts"] + }, + "dependencies": { + "@types/aws-lambda": "^8.10.76" + }, + "devDependencies": { + "esbuild": "^0.11.23" + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/excluded.js b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/excluded.js new file mode 100644 index 000000000..8bf8be437 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/excluded.js @@ -0,0 +1,2 @@ +//excluded +const x = 1; diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/included.js b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/included.js new file mode 100644 index 000000000..6d43fd9f0 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/included.js @@ -0,0 +1,6 @@ +//included +const request = require('minimal-request-promise'); +exports.handler = async (event, context) => { + const result = await(request.get(event.url)); + return request; +}; diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/package-lock.json b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/package-lock.json new file mode 100644 index 000000000..93a7f9fd2 --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/package-lock.json @@ -0,0 +1,47 @@ +{ + "name": "with-deps-esbuild", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "with-deps-esbuild", + "version": "1.0.0", + "license": "APACHE2.0", + "dependencies": { + "minimal-request-promise": "^1.5.0" + }, + "devDependencies": { + "esbuild": "^0.11.23" + } + }, + "node_modules/esbuild": { + "version": "0.11.23", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.23.tgz", + "integrity": "sha512-iaiZZ9vUF5wJV8ob1tl+5aJTrwDczlvGP0JoMmnpC2B0ppiMCu8n8gmy5ZTGl5bcG081XBVn+U+jP+mPFm5T5Q==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + } + }, + "node_modules/minimal-request-promise": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/minimal-request-promise/-/minimal-request-promise-1.5.0.tgz", + "integrity": "sha1-YPXX9VtAJtGXB04uFVYm1MxcLrw=" + } + }, + "dependencies": { + "esbuild": { + "version": "0.11.23", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.23.tgz", + "integrity": "sha512-iaiZZ9vUF5wJV8ob1tl+5aJTrwDczlvGP0JoMmnpC2B0ppiMCu8n8gmy5ZTGl5bcG081XBVn+U+jP+mPFm5T5Q==", + "dev": true + }, + "minimal-request-promise": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/minimal-request-promise/-/minimal-request-promise-1.5.0.tgz", + "integrity": "sha1-YPXX9VtAJtGXB04uFVYm1MxcLrw=" + } + } +} diff --git a/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/package.json b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/package.json new file mode 100644 index 000000000..c4f10260c --- /dev/null +++ b/tests/integration/workflows/nodejs_npm/testdata/with-deps-esbuild/package.json @@ -0,0 +1,18 @@ +{ + "name": "with-deps-esbuild", + "version": "1.0.0", + "description": "", + "keywords": [], + "author": "", + "license": "APACHE2.0", + "aws_sam": { + "bundler": "esbuild", + "entry_points": ["included.js"] + }, + "dependencies": { + "minimal-request-promise": "^1.5.0" + }, + "devDependencies": { + "esbuild": "^0.11.23" + } +} diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 5983a1810..2dd9b63ed 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -1,3 +1,4 @@ +import itertools from unittest import TestCase from mock import patch, call, Mock from parameterized import parameterized, param @@ -108,7 +109,7 @@ def __init__( self.assertEqual(builder.selected_workflow_cls, MyWorkflow) -class TesetLambdaBuilder_build(TestCase): +class TestLambdaBuilder_build(TestCase): def tearDown(self): # we don't want test classes lurking around and interfere with other tests DEFAULT_REGISTRY.clear() @@ -118,11 +119,27 @@ def setUp(self): self.lang_framework = "pip" self.app_framework = "chalice" - @parameterized.expand([param(True), param(False)]) + @parameterized.expand( + itertools.product( + [True, False], # scratch_dir_exists + [True, False], # download_dependencies + [None, "dependency_dir"], # dependency_dir + [True, False], # combine_dependencies + [None, [], ["a", "b"]], # experimental flags + ) + ) @patch("aws_lambda_builders.builder.os") - @patch("aws_lambda_builders.builder.importlib") @patch("aws_lambda_builders.builder.get_workflow") - def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, os_mock): + def test_with_mocks( + self, + scratch_dir_exists, + download_dependencies, + dependency_dir, + combine_dependencies, + experimental_flags, + get_workflow_mock, + os_mock, + ): workflow_cls = Mock() workflow_instance = workflow_cls.return_value = Mock() @@ -143,9 +160,10 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, options="options", executable_search_paths="executable_search_paths", mode=None, - download_dependencies=False, - dependencies_dir="dependency_folder", - combine_dependencies=False, + download_dependencies=download_dependencies, + dependencies_dir=dependency_dir, + combine_dependencies=combine_dependencies, + experimental_flags=experimental_flags, ) workflow_cls.assert_called_with( @@ -159,9 +177,10 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, options="options", executable_search_paths="executable_search_paths", mode=None, - download_dependencies=False, - dependencies_dir="dependency_folder", - combine_dependencies=False, + download_dependencies=download_dependencies, + dependencies_dir=dependency_dir, + combine_dependencies=combine_dependencies, + experimental_flags=experimental_flags, ) workflow_instance.run.assert_called_once() os_mock.path.exists.assert_called_once_with("scratch_dir") diff --git a/tests/unit/workflows/nodejs_npm/test_actions.py b/tests/unit/workflows/nodejs_npm/test_actions.py index 39007a59c..f1b48ab66 100644 --- a/tests/unit/workflows/nodejs_npm/test_actions.py +++ b/tests/unit/workflows/nodejs_npm/test_actions.py @@ -7,6 +7,9 @@ NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction, + NodejsNpmLockFileCleanUpAction, + NodejsNpmCIAction, + EsbuildBundleAction, ) from aws_lambda_builders.workflows.nodejs_npm.npm import NpmExecutionError @@ -54,7 +57,7 @@ def test_raises_action_failed_when_npm_fails(self, OSUtilMock, SubprocessNpmMock class TestNodejsNpmInstallAction(TestCase): @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") - def test_tars_and_unpacks_npm_project(self, SubprocessNpmMock): + def test_installs_npm_production_dependencies_for_npm_project(self, SubprocessNpmMock): subprocess_npm = SubprocessNpmMock.return_value action = NodejsNpmInstallAction("artifacts", subprocess_npm=subprocess_npm) @@ -65,6 +68,18 @@ def test_tars_and_unpacks_npm_project(self, SubprocessNpmMock): subprocess_npm.run.assert_called_with(expected_args, cwd="artifacts") + @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") + def test_can_set_mode(self, SubprocessNpmMock): + subprocess_npm = SubprocessNpmMock.return_value + + action = NodejsNpmInstallAction("artifacts", subprocess_npm=subprocess_npm, is_production=False) + + action.execute() + + expected_args = ["install", "-q", "--no-audit", "--no-save", "--production=false", "--unsafe-perm"] + + subprocess_npm.run.assert_called_with(expected_args, cwd="artifacts") + @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") def test_raises_action_failed_when_npm_fails(self, SubprocessNpmMock): subprocess_npm = SubprocessNpmMock.return_value @@ -80,6 +95,32 @@ def test_raises_action_failed_when_npm_fails(self, SubprocessNpmMock): self.assertEqual(raised.exception.args[0], "NPM Failed: boom!") +class TestNodejsNpmCIAction(TestCase): + @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") + def test_tars_and_unpacks_npm_project(self, SubprocessNpmMock): + subprocess_npm = SubprocessNpmMock.return_value + + action = NodejsNpmCIAction("sources", subprocess_npm=subprocess_npm) + + action.execute() + + subprocess_npm.run.assert_called_with(["ci"], cwd="sources") + + @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") + def test_raises_action_failed_when_npm_fails(self, SubprocessNpmMock): + subprocess_npm = SubprocessNpmMock.return_value + + builder_instance = SubprocessNpmMock.return_value + builder_instance.run.side_effect = NpmExecutionError(message="boom!") + + action = NodejsNpmCIAction("sources", subprocess_npm=subprocess_npm) + + with self.assertRaises(ActionFailedError) as raised: + action.execute() + + self.assertEqual(raised.exception.args[0], "NPM Failed: boom!") + + class TestNodejsNpmrcCopyAction(TestCase): @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") def test_copies_npmrc_into_a_project(self, OSUtilMock): @@ -140,3 +181,219 @@ def test_skips_npmrc_removal_if_npmrc_doesnt_exist(self, OSUtilMock): action.execute() osutils.remove_file.assert_not_called() + + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_raises_action_failed_when_removing_fails(self, OSUtilMock): + osutils = OSUtilMock.return_value + osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) + + osutils.remove_file.side_effect = OSError() + + action = NodejsNpmrcCleanUpAction("artifacts", osutils=osutils) + + with self.assertRaises(ActionFailedError): + action.execute() + + +class TestNodejsNpmLockFileCleanUpAction(TestCase): + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_removes_dot_package_lock_if_exists(self, OSUtilMock): + osutils = OSUtilMock.return_value + osutils.joinpath.side_effect = lambda a, b, c: "{}/{}/{}".format(a, b, c) + + action = NodejsNpmLockFileCleanUpAction("artifacts", osutils=osutils) + osutils.file_exists.side_effect = [True] + action.execute() + + osutils.remove_file.assert_called_with("artifacts/node_modules/.package-lock.json") + + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_skips_lockfile_removal_if_it_doesnt_exist(self, OSUtilMock): + osutils = OSUtilMock.return_value + osutils.joinpath.side_effect = lambda a, b, c: "{}/{}/{}".format(a, b, c) + + action = NodejsNpmLockFileCleanUpAction("artifacts", osutils=osutils) + osutils.file_exists.side_effect = [False] + action.execute() + + osutils.remove_file.assert_not_called() + + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_raises_action_failed_when_removing_fails(self, OSUtilMock): + osutils = OSUtilMock.return_value + osutils.joinpath.side_effect = lambda a, b, c: "{}/{}/{}".format(a, b, c) + + osutils.remove_file.side_effect = OSError() + + action = NodejsNpmLockFileCleanUpAction("artifacts", osutils=osutils) + + with self.assertRaises(ActionFailedError): + action.execute() + + +class TestEsbuildBundleAction(TestCase): + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + @patch("aws_lambda_builders.workflows.nodejs_npm.esbuild.SubprocessEsbuild") + def setUp(self, OSUtilMock, SubprocessEsbuildMock): + self.osutils = OSUtilMock.return_value + self.subprocess_esbuild = SubprocessEsbuildMock.return_value + self.osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) + self.osutils.file_exists.side_effect = [True, True] + + def test_raises_error_if_entrypoints_not_specified(self): + action = EsbuildBundleAction("source", "artifacts", {"config": "param"}, self.osutils, self.subprocess_esbuild) + with self.assertRaises(ActionFailedError) as raised: + action.execute() + + self.assertEqual(raised.exception.args[0], "entry_points not set ({'config': 'param'})") + + def test_raises_error_if_entrypoints_not_a_list(self): + action = EsbuildBundleAction( + "source", "artifacts", {"config": "param", "entry_points": "abc"}, self.osutils, self.subprocess_esbuild + ) + with self.assertRaises(ActionFailedError) as raised: + action.execute() + + self.assertEqual( + raised.exception.args[0], "entry_points must be a list ({'config': 'param', 'entry_points': 'abc'})" + ) + + def test_raises_error_if_entrypoints_empty_list(self): + action = EsbuildBundleAction( + "source", "artifacts", {"config": "param", "entry_points": []}, self.osutils, self.subprocess_esbuild + ) + with self.assertRaises(ActionFailedError) as raised: + action.execute() + + self.assertEqual( + raised.exception.args[0], "entry_points must not be empty ({'config': 'param', 'entry_points': []})" + ) + + def test_packages_javascript_with_minification_and_sourcemap(self): + action = EsbuildBundleAction( + "source", "artifacts", {"entry_points": ["x.js"]}, self.osutils, self.subprocess_esbuild + ) + action.execute() + + self.subprocess_esbuild.run.assert_called_with( + [ + "x.js", + "--bundle", + "--platform=node", + "--format=cjs", + "--minify", + "--sourcemap", + "--target=es2020", + "--outdir=artifacts", + ], + cwd="source", + ) + + def test_checks_if_single_entrypoint_exists(self): + + action = EsbuildBundleAction( + "source", "artifacts", {"entry_points": ["x.js"]}, self.osutils, self.subprocess_esbuild + ) + self.osutils.file_exists.side_effect = [False] + + with self.assertRaises(ActionFailedError) as raised: + action.execute() + + self.osutils.file_exists.assert_called_with("source/x.js") + + self.assertEqual(raised.exception.args[0], "entry point source/x.js does not exist") + + def test_checks_if_multiple_entrypoints_exist(self): + + self.osutils.file_exists.side_effect = [True, False] + action = EsbuildBundleAction( + "source", "artifacts", {"entry_points": ["x.js", "y.js"]}, self.osutils, self.subprocess_esbuild + ) + + with self.assertRaises(ActionFailedError) as raised: + action.execute() + + self.osutils.file_exists.assert_any_call("source/x.js") + + self.osutils.file_exists.assert_called_with("source/y.js") + + self.assertEqual(raised.exception.args[0], "entry point source/y.js does not exist") + + def test_excludes_sourcemap_if_requested(self): + action = EsbuildBundleAction( + "source", "artifacts", {"entry_points": ["x.js"], "sourcemap": False}, self.osutils, self.subprocess_esbuild + ) + action.execute() + self.subprocess_esbuild.run.assert_called_with( + [ + "x.js", + "--bundle", + "--platform=node", + "--format=cjs", + "--minify", + "--target=es2020", + "--outdir=artifacts", + ], + cwd="source", + ) + + def test_does_not_minify_if_requested(self): + action = EsbuildBundleAction( + "source", "artifacts", {"entry_points": ["x.js"], "minify": False}, self.osutils, self.subprocess_esbuild + ) + action.execute() + self.subprocess_esbuild.run.assert_called_with( + [ + "x.js", + "--bundle", + "--platform=node", + "--format=cjs", + "--sourcemap", + "--target=es2020", + "--outdir=artifacts", + ], + cwd="source", + ) + + def test_uses_specified_target(self): + action = EsbuildBundleAction( + "source", "artifacts", {"entry_points": ["x.js"], "target": "node14"}, self.osutils, self.subprocess_esbuild + ) + action.execute() + self.subprocess_esbuild.run.assert_called_with( + [ + "x.js", + "--bundle", + "--platform=node", + "--format=cjs", + "--minify", + "--sourcemap", + "--target=node14", + "--outdir=artifacts", + ], + cwd="source", + ) + + def test_includes_multiple_entry_points_if_requested(self): + action = EsbuildBundleAction( + "source", + "artifacts", + {"entry_points": ["x.js", "y.js"], "target": "node14"}, + self.osutils, + self.subprocess_esbuild, + ) + action.execute() + self.subprocess_esbuild.run.assert_called_with( + [ + "x.js", + "y.js", + "--bundle", + "--platform=node", + "--format=cjs", + "--minify", + "--sourcemap", + "--target=node14", + "--outdir=artifacts", + ], + cwd="source", + ) diff --git a/tests/unit/workflows/nodejs_npm/test_esbuild.py b/tests/unit/workflows/nodejs_npm/test_esbuild.py new file mode 100644 index 000000000..4a3f6493b --- /dev/null +++ b/tests/unit/workflows/nodejs_npm/test_esbuild.py @@ -0,0 +1,79 @@ +from unittest import TestCase +from mock import patch + +from aws_lambda_builders.workflows.nodejs_npm.esbuild import SubprocessEsbuild, EsbuildExecutionError + + +class FakePopen: + def __init__(self, out=b"out", err=b"err", retcode=0): + self.out = out + self.err = err + self.returncode = retcode + + def communicate(self): + return self.out, self.err + + +class TestSubprocessEsbuild(TestCase): + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def setUp(self, OSUtilMock): + self.osutils = OSUtilMock.return_value + self.osutils.pipe = "PIPE" + self.popen = FakePopen() + self.osutils.popen.side_effect = [self.popen] + + which = lambda cmd, executable_search_paths: ["{}/{}".format(executable_search_paths[0], cmd)] + + self.under_test = SubprocessEsbuild(self.osutils, ["/a/b", "/c/d"], which) + + def test_run_executes_binary_found_in_exec_paths(self): + + self.under_test.run(["arg-a", "arg-b"]) + + self.osutils.popen.assert_called_with( + ["/a/b/esbuild", "arg-a", "arg-b"], cwd=None, stderr="PIPE", stdout="PIPE" + ) + + def test_uses_cwd_if_supplied(self): + self.under_test.run(["arg-a", "arg-b"], cwd="/a/cwd") + + self.osutils.popen.assert_called_with( + ["/a/b/esbuild", "arg-a", "arg-b"], cwd="/a/cwd", stderr="PIPE", stdout="PIPE" + ) + + def test_returns_popen_out_decoded_if_retcode_is_0(self): + self.popen.out = b"some encoded text\n\n" + + result = self.under_test.run(["pack"]) + + self.assertEqual(result, "some encoded text") + + def test_raises_EsbuildExecutionError_with_err_text_if_retcode_is_not_0(self): + self.popen.returncode = 1 + self.popen.err = b"some error text\n\n" + + with self.assertRaises(EsbuildExecutionError) as raised: + self.under_test.run(["pack"]) + + self.assertEqual(raised.exception.args[0], "Esbuild Failed: some error text") + + def test_raises_EsbuildExecutionError_if_which_returns_no_results(self): + + which = lambda cmd, executable_search_paths: [] + self.under_test = SubprocessEsbuild(self.osutils, ["/a/b", "/c/d"], which) + with self.assertRaises(EsbuildExecutionError) as raised: + self.under_test.run(["pack"]) + + self.assertEqual(raised.exception.args[0], "Esbuild Failed: cannot find esbuild") + + def test_raises_ValueError_if_args_not_a_list(self): + with self.assertRaises(ValueError) as raised: + self.under_test.run(("pack")) + + self.assertEqual(raised.exception.args[0], "args must be a list") + + def test_raises_ValueError_if_args_empty(self): + with self.assertRaises(ValueError) as raised: + self.under_test.run([]) + + self.assertEqual(raised.exception.args[0], "requires at least one arg") diff --git a/tests/unit/workflows/nodejs_npm/test_utils.py b/tests/unit/workflows/nodejs_npm/test_utils.py new file mode 100644 index 000000000..80d92a9a5 --- /dev/null +++ b/tests/unit/workflows/nodejs_npm/test_utils.py @@ -0,0 +1,17 @@ +from unittest import TestCase +from parameterized import parameterized + +from aws_lambda_builders.workflows.nodejs_npm.utils import EXPERIMENTAL_FLAG_ESBUILD, is_experimental_esbuild_scope + + +class TestNodejsUtils(TestCase): + @parameterized.expand( + [ + (None, False), + ([], False), + ([EXPERIMENTAL_FLAG_ESBUILD], True), + ([EXPERIMENTAL_FLAG_ESBUILD, "SomeOtherFlag"], True), + ] + ) + def test_experimental_esbuild_scope_check(self, experimental_flags, expected): + self.assertEqual(is_experimental_esbuild_scope(experimental_flags), expected) diff --git a/tests/unit/workflows/nodejs_npm/test_workflow.py b/tests/unit/workflows/nodejs_npm/test_workflow.py index c47938fd0..9d2469f70 100644 --- a/tests/unit/workflows/nodejs_npm/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm/test_workflow.py @@ -1,17 +1,31 @@ -import mock - from unittest import TestCase +from mock import patch, call -from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction, MoveDependenciesAction, CleanUpAction +from aws_lambda_builders.exceptions import WorkflowFailedError +from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, CopyDependenciesAction, MoveDependenciesAction from aws_lambda_builders.architecture import ARM64 from aws_lambda_builders.workflows.nodejs_npm.workflow import NodejsNpmWorkflow +from aws_lambda_builders.workflows.nodejs_npm.esbuild import SubprocessEsbuild from aws_lambda_builders.workflows.nodejs_npm.actions import ( NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction, + NodejsNpmLockFileCleanUpAction, + NodejsNpmCIAction, + EsbuildBundleAction, ) -from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils +from aws_lambda_builders.workflows.nodejs_npm.utils import EXPERIMENTAL_FLAG_ESBUILD + + +class FakePopen: + def __init__(self, out=b"out", err=b"err", retcode=0): + self.out = out + self.err = err + self.returncode = retcode + + def communicate(self): + return self.out, self.err class TestNodejsNpmWorkflow(TestCase): @@ -21,25 +35,30 @@ class TestNodejsNpmWorkflow(TestCase): this is just a quick wiring test to provide fast feedback if things are badly broken """ - def setUp(self): - self.osutils_mock = mock.Mock(spec=OSUtils()) + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def setUp(self, OSUtilMock): + self.osutils = OSUtilMock.return_value + self.osutils.pipe = "PIPE" + self.popen = FakePopen() + self.osutils.popen.side_effect = [self.popen] + self.osutils.is_windows.side_effect = [False] + self.osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) def test_workflow_sets_up_npm_actions_with_download_dependencies_without_dependencies_dir(self): + self.osutils.file_exists.return_value = True - self.osutils_mock.file_exists.return_value = True - - workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils_mock) + workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils) - self.assertEqual(len(workflow.actions), 5) + self.assertEqual(len(workflow.actions), 6) self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) self.assertIsInstance(workflow.actions[2], CopySourceAction) self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) self.assertIsInstance(workflow.actions[4], NodejsNpmrcCleanUpAction) + self.assertIsInstance(workflow.actions[5], NodejsNpmLockFileCleanUpAction) def test_workflow_sets_up_npm_actions_without_download_dependencies_with_dependencies_dir(self): - - self.osutils_mock.file_exists.return_value = True + self.osutils.file_exists.return_value = True workflow = NodejsNpmWorkflow( "source", @@ -48,20 +67,35 @@ def test_workflow_sets_up_npm_actions_without_download_dependencies_with_depende "manifest", dependencies_dir="dep", download_dependencies=False, - osutils=self.osutils_mock, + osutils=self.osutils, ) - self.assertEqual(len(workflow.actions), 5) + self.assertEqual(len(workflow.actions), 7) self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) self.assertIsInstance(workflow.actions[2], CopySourceAction) self.assertIsInstance(workflow.actions[3], CopySourceAction) self.assertIsInstance(workflow.actions[4], NodejsNpmrcCleanUpAction) + self.assertIsInstance(workflow.actions[5], NodejsNpmLockFileCleanUpAction) + self.assertIsInstance(workflow.actions[6], NodejsNpmLockFileCleanUpAction) + + def test_workflow_sets_up_npm_actions_without_bundler_if_manifest_doesnt_request_it(self): + + workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils) + + self.assertEqual(len(workflow.actions), 6) + + self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) + self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) + self.assertIsInstance(workflow.actions[2], CopySourceAction) + self.assertIsInstance(workflow.actions[3], NodejsNpmInstallAction) + self.assertIsInstance(workflow.actions[4], NodejsNpmrcCleanUpAction) + self.assertIsInstance(workflow.actions[5], NodejsNpmLockFileCleanUpAction) def test_workflow_sets_up_npm_actions_with_download_dependencies_and_dependencies_dir(self): - self.osutils_mock.file_exists.return_value = True + self.osutils.file_exists.return_value = True workflow = NodejsNpmWorkflow( "source", @@ -70,10 +104,10 @@ def test_workflow_sets_up_npm_actions_with_download_dependencies_and_dependencie "manifest", dependencies_dir="dep", download_dependencies=True, - osutils=self.osutils_mock, + osutils=self.osutils, ) - self.assertEqual(len(workflow.actions), 7) + self.assertEqual(len(workflow.actions), 9) self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) @@ -82,11 +116,10 @@ def test_workflow_sets_up_npm_actions_with_download_dependencies_and_dependencie self.assertIsInstance(workflow.actions[4], CleanUpAction) self.assertIsInstance(workflow.actions[5], CopyDependenciesAction) self.assertIsInstance(workflow.actions[6], NodejsNpmrcCleanUpAction) + self.assertIsInstance(workflow.actions[7], NodejsNpmLockFileCleanUpAction) + self.assertIsInstance(workflow.actions[8], NodejsNpmLockFileCleanUpAction) def test_workflow_sets_up_npm_actions_without_download_dependencies_and_without_dependencies_dir(self): - - self.osutils_mock.file_exists.return_value = True - workflow = NodejsNpmWorkflow( "source", "artifacts", @@ -94,19 +127,20 @@ def test_workflow_sets_up_npm_actions_without_download_dependencies_and_without_ "manifest", dependencies_dir=None, download_dependencies=False, - osutils=self.osutils_mock, + osutils=self.osutils, ) - self.assertEqual(len(workflow.actions), 4) + self.assertEqual(len(workflow.actions), 5) self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) self.assertIsInstance(workflow.actions[2], CopySourceAction) self.assertIsInstance(workflow.actions[3], NodejsNpmrcCleanUpAction) + self.assertIsInstance(workflow.actions[4], NodejsNpmLockFileCleanUpAction) def test_workflow_sets_up_npm_actions_without_combine_dependencies(self): - self.osutils_mock.file_exists.return_value = True + self.osutils.file_exists.return_value = True workflow = NodejsNpmWorkflow( "source", @@ -116,10 +150,10 @@ def test_workflow_sets_up_npm_actions_without_combine_dependencies(self): dependencies_dir="dep", download_dependencies=True, combine_dependencies=False, - osutils=self.osutils_mock, + osutils=self.osutils, ) - self.assertEqual(len(workflow.actions), 7) + self.assertEqual(len(workflow.actions), 9) self.assertIsInstance(workflow.actions[0], NodejsNpmPackAction) self.assertIsInstance(workflow.actions[1], NodejsNpmrcCopyAction) @@ -128,34 +162,155 @@ def test_workflow_sets_up_npm_actions_without_combine_dependencies(self): self.assertIsInstance(workflow.actions[4], CleanUpAction) self.assertIsInstance(workflow.actions[5], MoveDependenciesAction) self.assertIsInstance(workflow.actions[6], NodejsNpmrcCleanUpAction) + self.assertIsInstance(workflow.actions[7], NodejsNpmLockFileCleanUpAction) + self.assertIsInstance(workflow.actions[8], NodejsNpmLockFileCleanUpAction) + + def test_workflow_sets_up_npm_actions_with_bundler_if_manifest_requests_it(self): + + self.osutils.parse_json.side_effect = [{"aws_sam": {"bundler": "esbuild"}}] + + self.osutils.file_exists.side_effect = [True, False, False] + + workflow = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + osutils=self.osutils, + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) + + self.assertEqual(len(workflow.actions), 2) + + self.assertIsInstance(workflow.actions[0], NodejsNpmInstallAction) + + self.assertIsInstance(workflow.actions[1], EsbuildBundleAction) + + self.osutils.parse_json.assert_called_with("manifest") + + self.osutils.file_exists.assert_has_calls( + [call("source/package-lock.json"), call("source/npm-shrinkwrap.json")] + ) + + def test_workflow_fails_if_manifest_parsing_fails(self): + + self.osutils.parse_json.side_effect = OSError("boom!") + + with self.assertRaises(WorkflowFailedError) as raised: + NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils) + + self.assertEqual(raised.exception.args[0], "NodejsNpmBuilder:ParseManifest - boom!") + + self.osutils.parse_json.assert_called_with("manifest") + + def test_sets_up_esbuild_search_path_from_npm_bin(self): + + self.popen.out = b"project/bin" + self.osutils.parse_json.side_effect = [{"aws_sam": {"bundler": "esbuild"}}] + + workflow = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + osutils=self.osutils, + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) + + self.osutils.popen.assert_called_with(["npm", "bin"], stdout="PIPE", stderr="PIPE", cwd="source") + + esbuild = workflow.actions[1].subprocess_esbuild + + self.assertIsInstance(esbuild, SubprocessEsbuild) + + self.assertEqual(esbuild.executable_search_paths, ["project/bin"]) + + def test_sets_up_esbuild_search_path_with_workflow_executable_search_paths_after_npm_bin(self): + + self.popen.out = b"project/bin" + self.osutils.parse_json.side_effect = [{"aws_sam": {"bundler": "esbuild"}}] - def test_workflow_only_copy_action(self): - self.osutils_mock.file_exists.return_value = False + workflow = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + osutils=self.osutils, + executable_search_paths=["other/bin"], + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) - workflow = NodejsNpmWorkflow("source", "artifacts", "scratch_dir", "manifest", osutils=self.osutils_mock) + self.osutils.popen.assert_called_with(["npm", "bin"], stdout="PIPE", stderr="PIPE", cwd="source") - self.assertEqual(len(workflow.actions), 1) + esbuild = workflow.actions[1].subprocess_esbuild - self.assertIsInstance(workflow.actions[0], CopySourceAction) + self.assertIsInstance(esbuild, SubprocessEsbuild) + + self.assertEqual(esbuild.executable_search_paths, ["project/bin", "other/bin"]) + + def test_workflow_uses_npm_ci_if_lockfile_exists(self): + + self.osutils.parse_json.side_effect = [{"aws_sam": {"bundler": "esbuild"}}] + self.osutils.file_exists.side_effect = [True, True] + + workflow = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + osutils=self.osutils, + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) + + self.assertEqual(len(workflow.actions), 2) + + self.assertIsInstance(workflow.actions[0], NodejsNpmCIAction) + + self.assertIsInstance(workflow.actions[1], EsbuildBundleAction) + + self.osutils.file_exists.assert_has_calls([call("source/package-lock.json")]) + + def test_workflow_uses_npm_ci_if_shrinkwrap_exists(self): + + self.osutils.parse_json.side_effect = [{"aws_sam": {"bundler": "esbuild"}}] + self.osutils.file_exists.side_effect = [True, False, True] + + workflow = NodejsNpmWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + osutils=self.osutils, + experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + ) + + self.assertEqual(len(workflow.actions), 2) + + self.assertIsInstance(workflow.actions[0], NodejsNpmCIAction) + + self.assertIsInstance(workflow.actions[1], EsbuildBundleAction) + + self.osutils.file_exists.assert_has_calls( + [call("source/package-lock.json"), call("source/npm-shrinkwrap.json")] + ) def test_must_validate_architecture(self): - self.osutils_mock.file_exists.return_value = True + self.osutils.is_windows.side_effect = [False, False] workflow = NodejsNpmWorkflow( "source", "artifacts", "scratch", "manifest", options={"artifact_executable_name": "foo"}, - osutils=self.osutils_mock, + osutils=self.osutils, ) workflow_with_arm = NodejsNpmWorkflow( "source", "artifacts", "scratch", "manifest", - options={"artifact_executable_name": "foo"}, architecture=ARM64, - osutils=self.osutils_mock, + osutils=self.osutils, ) self.assertEqual(workflow.architecture, "x86_64")