feat: update gitlab pipeline (#578)

* add `ap-southeast-5`

* revamp pipeline

* build image first

* awscli is already installed

* only build image on changes on dockerfile

* see if cli updates

* build image and check latest `aws-cli` version

* specify aws cli version

* update image building and aws cli install

.gitignore

@@ -10,4 +10,4 @@ package-lock.json
 **/.serverless
 /.idea/
 
-ci/build-*.yaml
+.gitlab/build-*.yaml

.gitlab-ci.yml

@@ -1,7 +1,26 @@
+variables:
+  DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-js
+  DOCKER_TARGET_VERSION: latest
+
 stages:
   - pre 
   - build
 
+ci image:
+  stage: build
+  image: registry.ddbuild.io/images/docker:20.10
+  tags: ["arch:arm64"]
+  needs: []
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"'
+      changes:
+        - .gitlab/Dockerfile
+      when: on_success
+  variables:
+    DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION}
+  script:
+    - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f .gitlab/Dockerfile .
+
 .go-cache: &go-cache
   key: datadog-lambda-js-go-cache
   policy: pull
@@ -13,16 +32,16 @@ generator:
   cache: *go-cache
   script:
     - apk add --no-cache gomplate
-    - gomplate --config ci/config.yaml
+    - gomplate --config .gitlab/config.yaml
   artifacts:
     paths:
-      - ci/*-pipeline.yaml
+      - .gitlab/*-pipeline.yaml
 
 build-layers:
   stage: build
   trigger:
     include:
-      - artifact: ci/build-pipeline.yaml
+      - artifact: .gitlab/build-pipeline.yaml
         job: generator
     strategy: depend
   rules:

.gitlab/Dockerfile

@@ -0,0 +1,18 @@
+FROM registry.ddbuild.io/images/docker:24.0.5
+
+RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
+  curl gcc gnupg g++ make cmake unzip openssl g++ uuid-runtime xxd ca-certificates
+
+# Install NodeJS 18.x
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN apt-get update && apt-get install -y nodejs
+
+# Install Yarn and Typescript globally
+RUN npm install --global yarn typescript
+RUN yarn global add serverless@^3 --prefix /usr/local
+
+# Install AWS CLI
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip && ./aws/install

.gitlab/config.yaml

@@ -0,0 +1,13 @@
+inputFiles:
+  - .gitlab/input_files/build.yaml.tpl
+
+outputFiles:
+  - .gitlab/build-pipeline.yaml
+
+datasources:
+  runtimes:
+    url: .gitlab/datasources/runtimes.yaml
+  regions:
+    url: .gitlab/datasources/regions.yaml
+  environments:
+    url: .gitlab/datasources/environments.yaml

ci/datasources/regions.yaml → .gitlab/datasources/regions.yaml

@@ -11,6 +11,7 @@ regions:
   - code: "ap-southeast-2"
   - code: "ap-southeast-3"
   - code: "ap-southeast-4"
+  - code: "ap-southeast-5"
   - code: "ap-northeast-1"
   - code: "ap-northeast-2"
   - code: "ap-northeast-3"

ci/input_files/build.yaml.tpl → .gitlab/input_files/build.yaml.tpl

@@ -1,3 +1,7 @@
+variables:
+  CI_DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-js
+  CI_DOCKER_TARGET_VERSION: latest
+
 stages:
  - build
  - test
@@ -11,18 +15,6 @@ default:
       # Retry when the runner fails to start
       - runner_system_failure
 
-.install-node: &install-node
-  - apt-get update
-  - apt-get install -y ca-certificates curl gnupg xxd
-  - mkdir -p /etc/apt/keyrings
-  - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
-  # We are explicitly setting the node_18.x version for the installation
-  - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
-  - apt-get update
-  - apt-get install nodejs -y
-  - npm install --global yarn
-  - npm install --global typescript
-
 .node-before-script: &node-before-script
   - echo 'yarn-offline-mirror ".yarn-cache/"' >> .yarnrc
   - echo 'yarn-offline-mirror-pruning true' >> .yarnrc
@@ -36,10 +28,10 @@ default:
     - $CI_PROJECT_DIR/.yarn-cache
   policy: pull
 
-build-layer ({{ $runtime.name }}):
+build layer ({{ $runtime.name }}):
   stage: build
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
   artifacts:
     expire_in: 1 hr # Unsigned zips expire in 1 hour
     paths:
@@ -49,14 +41,14 @@ build-layer ({{ $runtime.name }}):
   script:
     - NODE_VERSION={{ $runtime.node_version }} ./scripts/build_layers.sh
 
-check-layer-size ({{ $runtime.name }}):
+check layer size ({{ $runtime.name }}):
   stage: test
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
   needs: 
-    - build-layer ({{ $runtime.name }})
+    - build layer ({{ $runtime.name }})
   dependencies:
-    - build-layer ({{ $runtime.name }})
+    - build layer ({{ $runtime.name }})
   script: 
     - NODE_VERSION={{ $runtime.node_version }} ./scripts/check_layer_size.sh
 
@@ -66,122 +58,119 @@ lint ({{ $runtime.name }}):
   image: registry.ddbuild.io/images/mirror/node:{{ $runtime.node_major_version }}-bullseye
   cache: &{{ $runtime.name }}-cache
   before_script: *node-before-script
+  needs: []
   script: 
     - yarn check-formatting
     - yarn lint
 
-unit-test ({{ $runtime.name }}):
+unit test ({{ $runtime.name }}):
   stage: test
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/mirror/node:{{ $runtime.node_major_version }}-bullseye
   cache: &{{ $runtime.name }}-cache
   before_script: *node-before-script
+  needs: []
   script: 
     - yarn build
     - yarn test --ci --forceExit --detectOpenHandles
     - bash <(curl -s https://codecov.io/bash)
 
-integration-test ({{ $runtime.name }}):
+integration test ({{ $runtime.name }}):
   stage: test
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
   needs: 
-    - build-layer ({{ $runtime.name }})
+    - build layer ({{ $runtime.name }})
   dependencies:
-    - build-layer ({{ $runtime.name }})
+    - build layer ({{ $runtime.name }})
   cache: &{{ $runtime.name }}-cache
   variables:
     CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
   before_script:
-    - *install-node
-    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
-    - yarn global add serverless@^3.38.0 --prefix /usr/local
+    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source .gitlab/scripts/get_secrets.sh
     - cd integration_tests && yarn install && cd ..
   script:
     - RUNTIME_PARAM={{ $runtime.node_major_version }} ./scripts/run_integration_tests.sh
 
 {{ range $environment := (ds "environments").environments }}
 
 {{ if or (eq $environment.name "prod") }}
-sign-layer ({{ $runtime.name }}):
+sign layer ({{ $runtime.name }}):
   stage: sign
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
   rules:
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
       when: manual
   needs:
-    - build-layer ({{ $runtime.name }})
-    - check-layer-size ({{ $runtime.name }})
+    - build layer ({{ $runtime.name }})
+    - check layer size ({{ $runtime.name }})
     - lint ({{ $runtime.name }})
-    - unit-test ({{ $runtime.name }})
-    - integration-test ({{ $runtime.name }})
+    - unit test ({{ $runtime.name }})
+    - integration test ({{ $runtime.name }})
   dependencies:
-    - build-layer ({{ $runtime.name }})
+    - build layer ({{ $runtime.name }})
   artifacts: # Re specify artifacts so the modified signed file is passed
     expire_in: 1 day # Signed layers should expire after 1 day
     paths:
       - .layers/datadog_lambda_node{{ $runtime.node_version }}.zip
   before_script:
-    - apt-get update
-    - apt-get install -y uuid-runtime
-    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
   script:
     - LAYER_FILE=datadog_lambda_node{{ $runtime.node_version }}.zip ./scripts/sign_layers.sh {{ $environment.name }}
 {{ end }}
 
-publish-layer-{{ $environment.name }} ({{ $runtime.name }}):
+publish layer {{ $environment.name }} ({{ $runtime.name }}):
   stage: publish
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
   rules:
     - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/'
       when: manual
       allow_failure: true
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
   needs:
 {{ if or (eq $environment.name "prod") }}
-      - sign-layer ({{ $runtime.name }})
+      - sign layer ({{ $runtime.name }})
 {{ else }}
-      - build-layer ({{ $runtime.name }})
-      - check-layer-size ({{ $runtime.name }})
+      - build layer ({{ $runtime.name }})
+      - check layer size ({{ $runtime.name }})
       - lint ({{ $runtime.name }})
-      - unit-test ({{ $runtime.name }})
-      - integration-test ({{ $runtime.name }})
+      - unit test ({{ $runtime.name }})
+      - integration test ({{ $runtime.name }})
 {{ end }}
   dependencies:
 {{ if or (eq $environment.name "prod") }}
-      - sign-layer ({{ $runtime.name }})
+      - sign layer ({{ $runtime.name }})
 {{ else }}
-      - build-layer ({{ $runtime.name }})
+      - build layer ({{ $runtime.name }})
 {{ end }}
   parallel:
     matrix:
       - REGION: {{ range (ds "regions").regions }}
           - {{ .code }}
         {{- end}}
   before_script:
-    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
+    - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
   script:
-    - STAGE={{ $environment.name }} NODE_VERSION={{ $runtime.node_version }} ./ci/publish_layers.sh
+    - STAGE={{ $environment.name }} NODE_VERSION={{ $runtime.node_version }} .gitlab/scripts/publish_layers.sh
 
 {{- end }}
 
 {{- end }}
 
-publish-npm-package:
+publish npm package:
   stage: publish
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
   cache: []
   rules:
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
   when: manual
   needs: {{ range $runtime := (ds "runtimes").runtimes }}
-    - sign-layer ({{ $runtime.name }})
+    - sign layer ({{ $runtime.name }})
   {{- end }}
   before_script:
-    - *install-node
     - *node-before-script
   script:
-    - ./ci/publish_npm.sh
+    - .gitlab/scripts/publish_npm.sh

ci/publish_layers.sh → .gitlab/scripts/publish_layers.sh

@@ -17,8 +17,6 @@ NODE_VERSIONS=("16.14" "18.12" "20.9")
 STAGES=('prod', 'sandbox', 'staging')
 
 printf "Starting script...\n\n"
-printf "Installing dependencies\n"
-pip install awscli
 
 publish_layer() {
     region=$1