From a2adc5344c1795201debd4cc612be31bdf39f384 Mon Sep 17 00:00:00 2001 From: Nicholas Hulston Date: Mon, 3 Mar 2025 11:04:12 -0500 Subject: [PATCH] Create bundle and publish to govcloud script (#627) --- .gitlab/input_files/build.yaml.tpl | 28 +++++++ .gitlab/scripts/publish_layers.sh | 14 ++-- scripts/publish_govcloud_layers.sh | 124 +++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 5 deletions(-) create mode 100755 scripts/publish_govcloud_layers.sh diff --git a/.gitlab/input_files/build.yaml.tpl b/.gitlab/input_files/build.yaml.tpl index 3775f396..737d746e 100644 --- a/.gitlab/input_files/build.yaml.tpl +++ b/.gitlab/input_files/build.yaml.tpl @@ -175,3 +175,31 @@ publish npm package: - *node-before-script script: - .gitlab/scripts/publish_npm.sh + +{{ range $environment := (ds "environments").environments }} + +{{ if eq $environment.name "prod" }}signed {{ end }}layer bundle: + stage: {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + tags: ["arch:amd64"] + rules: + - if: '"{{ $environment.name }}" =~ /^sandbox/' + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: + {{ range $runtime := (ds "runtimes").runtimes }} + - {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} layer ({{ $runtime.name }}) + {{ end }} + dependencies: + {{ range $runtime := (ds "runtimes").runtimes }} + - {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} layer ({{ $runtime.name }}) + {{ end }} + artifacts: + expire_in: 1 day + paths: + - datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID}/ + name: datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + script: + - rm -rf datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + - mkdir -p datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + - cp .layers/datadog_lambda_node*.zip datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} +{{ end }} diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index 604ca748..97a38f17 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -95,14 +95,18 @@ if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then else # Running on prod if [ -z "$CI_COMMIT_TAG" ]; then - printf "[Error] No CI_COMMIT_TAG found.\n" - printf "Exiting script...\n" - exit 1 + # this happens during manual govcloud releases. + if [ -z "$VERSION" ]; then + printf "[Error] No CI_COMMIT_TAG or VERSION found.\n" + printf "Exiting script...\n" + exit 1 + else + printf "Using provided VERSION: $VERSION\n" + fi else printf "Tag found in environment: $CI_COMMIT_TAG\n" + VERSION=$(echo "${CI_COMMIT_TAG##*v}" | cut -d. -f2) fi - - VERSION=$(echo "${CI_COMMIT_TAG##*v}" | cut -d. -f2) fi # Target layer version diff --git a/scripts/publish_govcloud_layers.sh b/scripts/publish_govcloud_layers.sh new file mode 100755 index 00000000..68e1725e --- /dev/null +++ b/scripts/publish_govcloud_layers.sh @@ -0,0 +1,124 @@ +#! /usr/bin/env bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2025 Datadog, Inc. +# +# USAGE: download the layer bundle from the build pipeline in gitlab. Use the +# Download button on the `layer bundle` job. This will be a zip file containing +# all of the required layers. Run this script as follows: +# +# ENVIRONMENT=[us1-staging-fed or us1-fed] [PIPELINE_LAYER_SUFFIX=optional-layer-suffix] [REGIONS=us-gov-west-1] ./scripts/publish_govcloud_layers.sh +# +# protip: you can drag the zip file from finder into your terminal to insert +# its path. + +set -e + +NODE_VERSIONS=("18.12" "20.9" "22.11") + +LAYER_PACKAGE=$1 + +if [ -z "$LAYER_PACKAGE" ]; then + printf "[ERROR]: layer package not provided\n" + exit 1 +fi + +PACKAGE_NAME=$(basename "$LAYER_PACKAGE" .zip) +echo package name: $PACKAGE_NAME + +if [ -z "$ENVIRONMENT" ]; then + printf "[ERROR]: ENVIRONMENT not specified\n" + exit 1 +fi + +if [ "$ENVIRONMENT" = "us1-staging-fed" ]; then + AWS_VAULT_ROLE=sso-govcloud-us1-staging-fed-power-user + +# this role looks like this in ~/.aws/config: +# [profile sso-govcloud-us1-staging-fed-power-user] +# sso_start_url=https://start.us-gov-home.awsapps.com/directory/d-9867188aeb +# sso_account_id=553727695824 +# sso_role_name=power-user +# sso_region=us-gov-west-1 +# region=us-gov-west-1 + + export STAGE="sandbox" + if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_js-(signed-)?bundle-[0-9]+$ ]]; then + echo "[ERROR]: Unexpected package name: $PACKAGE_NAME" + exit 1 + fi + +elif [ $ENVIRONMENT = "us1-fed" ]; then + AWS_VAULT_ROLE=sso-govcloud-us1-fed-engineering + +# this role looks like this in ~/.aws/config: +# [profile sso-govcloud-us1-fed-engineering] +# sso_start_url=https://start.us-gov-west-1.us-gov-home.awsapps.com/directory/d-98671fdc8b +# sso_account_id=002406178527 +# sso_role_name=engineering +# sso_region=us-gov-west-1 +# region=us-gov-west-1 + + export STAGE="prod" + if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_js-signed-bundle-[0-9]+$ ]]; then + echo "[ERROR]: Unexpected package name: $PACKAGE_NAME" + exit 1 + fi + +else + printf "[ERROR]: ENVIRONMENT not supported, must be us1-staging-fed or us1-fed.\n" + exit 1 +fi + +# Clean and recreate the .layers directory +echo "Cleaning .layers directory..." +rm -rf .layers +mkdir -p .layers + +echo "Copying layer files to .layers directory..." +TEMP_DIR=$(mktemp -d) +unzip $LAYER_PACKAGE -d $TEMP_DIR +cp -v $TEMP_DIR/$PACKAGE_NAME/*.zip .layers/ + + +AWS_VAULT_PREFIX="aws-vault exec $AWS_VAULT_ROLE --" + +echo "Checking that you have access to the GovCloud AWS account" +$AWS_VAULT_PREFIX aws sts get-caller-identity + + +AVAILABLE_REGIONS=$($AWS_VAULT_PREFIX aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') + +# Determine the target regions +if [ -z "$REGIONS" ]; then + echo "Region not specified, running for all available regions." + REGIONS=$AVAILABLE_REGIONS +else + echo "Region specified: $REGIONS" + if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then + echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS" + echo "" + echo "EXITING SCRIPT." + exit 1 + fi +fi + +for region in $REGIONS +do + echo "Starting publishing layers for region $region..." + + for NODE_VERSION in "${NODE_VERSIONS[@]}"; do + echo "Publishing Layer for Node ${NODE_VERSION} in region ${region}" + + # Set environment variables for the publish script + export REGION=$region + export NODE_VERSION=$NODE_VERSION + + # Run the publish script with AWS credentials + $AWS_VAULT_PREFIX .gitlab/scripts/publish_layers.sh + done +done + +echo "Done!" \ No newline at end of file