chore: Automate Releases

.gitignore

@@ -9,3 +9,5 @@ package-lock.json
 
 **/.serverless
 /.idea/
+
+ci/build-*.yaml

.gitlab-ci.yml

@@ -0,0 +1,31 @@
+stages:
+  - pre 
+  - build
+
+.go-cache: &go-cache
+  key: datadog-lambda-js-go-cache
+  paths:
+    - $CI_PROJECT_DIR/.go-cache/pkg/mod
+  policy: pull
+
+generator:
+  stage: pre
+  image: registry.ddbuild.io/images/mirror/golang:alpine
+  tags: ["arch:amd64"]
+  cache: *go-cache
+  script:
+    - go install github.com/hairyhenderson/gomplate/v4/cmd/gomplate@latest
+    - gomplate --config ci/config.yaml
+  artifacts:
+    paths:
+      - ci/*-pipeline.yaml
+
+build-layers:
+  stage: build
+  trigger:
+    include:
+      - artifact: ci/build-pipeline.yaml
+        job: generator
+    strategy: depend
+  rules:
+    - when: on_success

Dockerfile

@@ -1,5 +1,5 @@
 ARG image
-FROM $image
+FROM $image as builder
 ARG image
 
 # Install git so we can specify a specific git ref (ie: refs/head/my-feature) in package.json 
@@ -55,4 +55,5 @@ RUN find /nodejs/node_modules -name "*.d.ts" -delete
 RUN find /nodejs/node_modules -name "*.js.map" -delete
 RUN find /nodejs/node_modules -name "*.ts.map" -delete
 
-
+FROM scratch
+COPY --from=builder /nodejs /

ci/config.yaml

@@ -0,0 +1,13 @@
+inputFiles:
+  - ci/input_files/build.yaml.tpl
+
+outputFiles:
+  - ci/build-pipeline.yaml
+
+datasources:
+  runtimes:
+    url: ci/datasources/runtimes.yaml
+  regions:
+    url: ci/datasources/regions.yaml
+  environments:
+    url: ci/datasources/environments.yaml

ci/datasources/environments.yaml

@@ -0,0 +1,9 @@
+environments:
+  - name: sandbox
+    external_id: sandbox-publish-externalid
+    role_to_assume: sandbox-layer-deployer
+    account: 425362996713
+  - name: prod
+    external_id: prod-publish-externalid
+    role_to_assume: dd-serverless-layer-deployer-role
+    account: 464622532012

ci/datasources/regions.yaml

@@ -0,0 +1,29 @@
+regions:
+  - code: "us-east-1"
+  - code: "us-east-2"
+  - code: "us-west-1"
+  - code: "us-west-2"
+  - code: "af-south-1"
+  - code: "ap-east-1"
+  - code: "ap-south-1"
+  - code: "ap-south-2"
+  - code: "ap-southeast-1"
+  - code: "ap-southeast-2"
+  - code: "ap-southeast-3"
+  - code: "ap-southeast-4"
+  - code: "ap-northeast-1"
+  - code: "ap-northeast-2"
+  - code: "ap-northeast-3"
+  - code: "ca-central-1"
+  - code: "ca-west-1"
+  - code: "eu-central-1"
+  - code: "eu-central-2"
+  - code: "eu-west-1"
+  - code: "eu-west-2"
+  - code: "eu-west-3"
+  - code: "eu-south-1"
+  - code: "eu-south-2"
+  - code: "il-central-1"
+  - code: "me-south-1"
+  - code: "me-central-1"
+  - code: "sa-east-1"

ci/datasources/runtimes.yaml

@@ -0,0 +1,13 @@
+runtimes:
+  - name: "node14"
+    node_version: "14.15"
+    node_major_version: "14"
+  - name: "node16"
+    node_version: "16.14"
+    node_major_version: "16"
+  - name: "node18"
+    node_version: "18.12"
+    node_major_version: "18"
+  - name: "node20"
+    node_version: "20.9"
+    node_major_version: "20"

ci/get_secrets.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2023 Datadog, Inc.
+
+set -e
+
+if [ -z "$EXTERNAL_ID_NAME" ]; then
+    printf "[Error] No EXTERNAL_ID_NAME found.\n"
+    printf "Exiting script...\n"
+    exit 1
+fi
+
+if [ -z "$ROLE_TO_ASSUME" ]; then
+    printf "[Error] No ROLE_TO_ASSUME found.\n"
+    printf "Exiting script...\n"
+    exit 1
+fi
+
+printf "Getting AWS External ID...\n"
+
+EXTERNAL_ID=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name "ci.datadog-lambda-js.$EXTERNAL_ID_NAME" \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
+printf "Getting DD API KEY...\n"
+
+export DD_API_KEY=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name ci.datadog-lambda-js.dd-api-key \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
+printf "Assuming role...\n"
+
+export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \
+    $(aws sts assume-role \
+    --role-arn "arn:aws:iam::$AWS_ACCOUNT:role/$ROLE_TO_ASSUME"  \
+    --role-session-name "ci.datadog-lambda-js-$CI_JOB_ID-$CI_JOB_STAGE" \
+    --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
+    --external-id $EXTERNAL_ID \
+    --output text))

ci/input_files/build.yaml.tpl

@@ -0,0 +1,177 @@
+stages:
+ - build
+ - test
+ - sign
+ - publish
+
+.install-node: &install-node
+  - apt-get update
+  - apt-get install -y ca-certificates curl gnupg xxd
+  - mkdir -p /etc/apt/keyrings
+  - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+  # We are explicitly setting the node_18.x version for the installation
+  - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+  - apt-get update
+  - apt-get install nodejs -y
+  - npm install --global yarn
+
+.node-before-script: &node-before-script
+  - echo 'yarn-offline-mirror ".yarn-cache/"' >> .yarnrc
+  - echo 'yarn-offline-mirror-pruning true' >> .yarnrc
+  - yarn install --frozen-lockfile --no-progress
+
+{{ $runtimes := (ds "runtimes").runtimes }}
+{{ range $runtime := $runtimes  }}
+
+.{{ $runtime.name }}-cache: &{{ $runtime.name }}-cache
+  key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
+  paths:
+    - $CI_PROJECT_DIR/.yarn-cache
+  policy: pull
+
+build-{{ $runtime.name }}-layer:
+  stage: build
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10
+  artifacts:
+    expire_in: 10 min # TODO: remove temp value
+    paths:
+      - .layers/datadog_lambda_node{{ $runtime.node_version }}.zip
+  variables:
+    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
+  script:
+    - NODE_VERSION={{ $runtime.node_version }} ./scripts/build_layers.sh
+
+check-{{ $runtime.name }}-layer-size:
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10
+  needs: 
+    - build-{{ $runtime.name }}-layer
+  dependencies:
+    - build-{{ $runtime.name }}-layer
+  script: 
+    - NODE_VERSION={{ $runtime.node_version }} ./scripts/check_layer_size.sh
+
+lint-{{ $runtime.name }}:
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/mirror/node:{{ $runtime.node_major_version }}-bullseye
+  cache: &{{ $runtime.name }}-cache
+  before_script: *node-before-script
+  script: 
+    - yarn check-formatting
+    - yarn lint
+
+unit-test-{{ $runtime.name }}:
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/mirror/node:{{ $runtime.node_major_version }}-bullseye
+  cache: &{{ $runtime.name }}-cache
+  before_script: *node-before-script
+  script: 
+    - yarn build
+    - yarn test --ci --forceExit --detectOpenHandles
+    - bash <(curl -s https://codecov.io/bash)
+
+integration-test-{{ $runtime.name }}:
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  needs: 
+    - build-{{ $runtime.name }}-layer
+  dependencies:
+    - build-{{ $runtime.name }}-layer
+  cache: &{{ $runtime.name }}-cache
+  variables:
+    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
+  before_script:
+    - *install-node
+    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
+    - yarn global add serverless --prefix /usr/local
+    - cd integration_tests && yarn install && cd ..
+  script:
+    - RUNTIME_PARAM={{ $runtime.node_major_version }} ./scripts/run_integration_tests.sh
+
+{{ $environments := (ds "environments").environments }}
+{{ range $environment := $environments }}
+
+{{ if or (eq $environment.name "prod") }}
+sign-{{ $environment.name }}-{{ $runtime.name }}-layer:
+  stage: sign
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+      when: manual
+  needs:
+    - build-{{ $runtime.name }}-layer
+    - check-{{ $runtime.name }}-layer-size
+    - lint-{{ $runtime.name }}
+    - unit-test-{{ $runtime.name }}
+    - integration-test-{{ $runtime.name }}
+  dependencies:
+    - build-{{ $runtime.name }}-layer
+  artifacts: # Re specify artifacts so the modified signed file is passed
+    expire_in: 10 min # TODO: remove temp value
+    paths:
+      - .layers/datadog_lambda_node{{ $runtime.node_version }}.zip
+  before_script:
+    - apt-get update
+    - apt-get install -y uuid-runtime
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
+  script:
+    - LAYER_FILE=datadog_lambda_node{{ $runtime.node_version }}.zip ./scripts/sign_layers.sh {{ $environment.name }}
+{{ end }}
+
+publish-{{ $environment.name }}-{{ $runtime.name }}-layer:
+  stage: publish
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  rules:
+    - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/'
+      when: manual
+      allow_failure: true
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+  needs:
+{{ if or (eq $environment.name "prod") }}
+      - sign-{{ $environment.name }}-{{ $runtime.name }}-layer
+{{ else }}
+      - build-{{ $runtime.name }}-layer
+      - check-{{ $runtime.name }}-layer-size
+      - lint-{{ $runtime.name }}
+      - unit-test-{{ $runtime.name }}
+      - integration-test-{{ $runtime.name }}
+{{ end }}
+  dependencies:
+{{ if or (eq $environment.name "prod") }}
+      - sign-{{ $environment.name }}-{{ $runtime.name }}-layer
+{{ else }}
+      - build-{{ $runtime.name }}-layer
+{{ end }}
+  parallel:
+    matrix:
+      - REGION: {{ range (ds "regions").regions }}
+          - {{ .code }}
+        {{- end}}
+  before_script:
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
+  script:
+    - STAGE={{ $environment.name }} NODE_VERSION={{ $runtime.node_version }} ./ci/publish_layers.sh
+
+{{- end }}
+
+{{- end }}
+
+publish-npm-package:
+  stage: publish
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  cache: []
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+  when: manual
+  before_script:
+    - *install-node
+  script:
+    - ./scripts/publish_npm.sh