feat: use multiple processes to publish layers (#438)

* feat: use multiple processes to publish layers * feat: Remove region arg
DataDog · Nov 17, 2023 · 9534c00 · joeyzhao2018 · Dec 8, 2023 · 9534c00
1 parent fbe67f7
commit 9534c00
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 20 deletions.
diff --git a/scripts/publish_layers.sh b/scripts/publish_layers.sh
@@ -14,6 +14,11 @@ NODE_VERSIONS_FOR_AWS_CLI=("nodejs14.x" "nodejs16.x" "nodejs18.x")
 LAYER_PATHS=(".layers/datadog_lambda_node14.15.zip" ".layers/datadog_lambda_node16.14.zip" ".layers/datadog_lambda_node18.12.zip")
 AVAILABLE_LAYERS=("Datadog-Node14-x" "Datadog-Node16-x" "Datadog-Node18-x")
 AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
+BATCH_SIZE=60
+PIDS=()
+
+# Makes sure any subprocesses will be terminated with this process
+trap "pkill -P $$; exit 1;" INT
 
 # Check that the layer files exist
 for layer_file in "${LAYER_PATHS[@]}"
@@ -99,6 +104,34 @@ publish_layer() {
     echo $version_nbr
 }
 
+wait_for_processes() {
+    for pid in "${PIDS[@]}"; do
+        wait $pid
+    done
+    PIDS=()
+}
+
+backfill_layers() {
+    latest_version=$1
+    region=$2
+    layer_name=$3
+    aws_version_key=$4
+    layer_path=$5
+    while [ $latest_version -lt $VERSION ]; do
+            latest_version=$(publish_layer $region $layer_name $aws_version_key $layer_path)
+            echo "Published version $latest_version for layer $layer_name in region $region"
+
+            # This shouldn't happen unless someone manually deleted the latest version, say 28, and
+            # then tries to republish 28 again. The published version would actually be 29, because
+            # Lambda layers are immutable and AWS will skip deleted version and use the next number.
+            if [ $latest_version -gt $VERSION ]; then
+                echo "ERROR: Published version $latest_version is greater than the desired version $VERSION!"
+                echo "Exiting"
+                exit 1
+            fi
+    done
+}
+
 for region in $REGIONS
 do
     echo "Starting publishing layer for region $region..."
@@ -109,31 +142,19 @@ do
             echo "Layer $layer_name version $VERSION already exists in region $region, skipping..."
             continue
         elif [ $latest_version -lt $((VERSION-1)) ]; then
-            read -p "WARNING: The latest version of layer $layer_name in region $region is $latest_version, publish all the missing versions including $VERSION or EXIT the script (y/n)?" CONT
-            if [ "$CONT" != "y" ]; then
-                echo "Exiting"
-                exit 1
-            fi
+            echo "WARNING: The latest version of layer $layer_name in region $region is $latest_version, this will publish all the missing versions including $VERSION"
         fi
 
         index=$(index_of_layer $layer_name)
         aws_version_key="${NODE_VERSIONS_FOR_AWS_CLI[$index]}"
         layer_path="${LAYER_PATHS[$index]}"
 
-        while [ $latest_version -lt $VERSION ]; do
-            latest_version=$(publish_layer $region $layer_name $aws_version_key $layer_path)
-            echo "Published version $latest_version for layer $layer_name in region $region"
-
-            # This shouldn't happen unless someone manually deleted the latest version, say 28, and
-            # then tries to republish 28 again. The published version would actually be 29, because
-            # Lambda layers are immutable and AWS will skip deleted version and use the next number.
-            if [ $latest_version -gt $VERSION ]; then
-                echo "ERROR: Published version $latest_version is greater than the desired version $VERSION!"
-                echo "Exiting"
-                exit 1
-            fi
-        done
+        backfill_layers $latest_version $region $layer_name $aws_version_key $layer_path &
+        PIDS+=($!)
+        if [ ${#PIDS[@]} -eq $BATCH_SIZE ]; then
+            wait_for_processes
+        fi
     done
 done
-
+wait_for_processes
 echo "Done !"
diff --git a/scripts/publish_sandbox.sh b/scripts/publish_sandbox.sh
@@ -10,7 +10,7 @@ if [ -z "$VERSION" ]; then
 fi
 
 ./scripts/build_layers.sh
-VERSION=$VERSION REGIONS=sa-east-1 ./scripts/publish_layers.sh
+VERSION=$VERSION ./scripts/publish_layers.sh
 
 # Automatically create PR against github.com/DataDog/documentation
 # If you'd like to test, please uncomment the below line