Merge pull request hashicorp#77 from hashicorp/ami-publishing

Change how we publish public AMIs

README.md

@@ -28,9 +28,9 @@ To deploy Consul servers using this Module:
    Here is an [example Packer template](https://github.com/hashicorp/terraform-aws-consul/tree/master/examples/consul-ami#quick-start). 
 
    If you are just experimenting with this Module, you may find it more convenient to use one of our official public AMIs:
-   - [Latest Ubuntu 16 AMIs](https://github.com/hashicorp/terraform-aws-consul/tree/master/_docs/ubuntu16-ami-list.md).
-   - [Latest Amazon Linux AMIs](https://github.com/hashicorp/terraform-aws-consul/tree/master/_docs/amazon-linux-ami-list.md).
-  
+   - **Latest Ubuntu 16 AMIs**: search for AMIs with names starting with `consul-ubuntu-` owned by account ID `562637147889`.
+   - **Latest Amazon Linux AMIs**: search for AMIs with names starting with `consul-amazon-linux-` owned by account ID `562637147889`.
+
     **WARNING! Do NOT use these AMIs in your production setup. In production, you should build your own AMIs in your own 
     AWS account.**
 

_ci/publish-amis.sh

@@ -7,51 +7,35 @@
 
 set -e
 
-readonly PACKER_TEMPLATE_PATH="/home/ubuntu/$CIRCLE_PROJECT_REPONAME/examples/consul-ami/consul.json"
-readonly PACKER_TEMPLATE_DEFAULT_REGION="us-east-1"
-readonly AMI_PROPERTIES_FILE="/tmp/ami.properties"
-readonly AMI_LIST_MARKDOWN_DIR="/home/ubuntu/$CIRCLE_PROJECT_REPONAME/_docs"
-readonly GIT_COMMIT_MESSAGE="Add latest AMI IDs."
-readonly GIT_USER_NAME="gruntwork-ci"
-readonly GIT_USER_EMAIL="[email protected]"
-
-# In CircleCI, every build populates the branch name in CIRCLE_BRANCH except builds triggered by a new tag, for which
-# the CIRCLE_BRANCH env var is empty. We assume tags are only issued against the master branch.
-readonly BRANCH_NAME="${CIRCLE_BRANCH:-master}"
-
-readonly PACKER_BUILD_NAME="$1"
-
-if [[ -z "$PACKER_BUILD_NAME" ]]; then
-  echo "ERROR: You must pass in the Packer build name as the first argument to this function."
+if [[ "$#" -ne 2 ]]; then
+  echo "Usage: publish-amis.sh PACKER_TEMPLATE_PATH BUILDER_NAME"
   exit 1
 fi
 
-echo "Checking out branch $BRANCH_NAME to make sure we do all work in a branch and not in detached HEAD state"
-git checkout "$BRANCH_NAME"
-
-# Build the example AMI. WARNING! In a production setting, you should build your own AMI to ensure it has exactly the
-# configuration you want. We build this example AMI solely to make initial use of this Module as easy as possible.
-build-packer-artifact \
-  --packer-template-path "$PACKER_TEMPLATE_PATH" \
-  --build-name "$PACKER_BUILD_NAME" \
-  --output-properties-file "$AMI_PROPERTIES_FILE"
-
-# Copy the AMI to all regions and make it public in each
-source "$AMI_PROPERTIES_FILE"
-publish-ami \
-  --all-regions \
-  --source-ami-id "$ARTIFACT_ID" \
-  --source-ami-region "$PACKER_TEMPLATE_DEFAULT_REGION" \
-  --output-markdown > "$AMI_LIST_MARKDOWN_DIR/$PACKER_BUILD_NAME-list.md" \
-  --markdown-title-text "$PACKER_BUILD_NAME: Latest Public AMIs" \
-  --markdown-description-text "**WARNING! Do NOT use these AMIs in a production setting.** They are meant only to make
-    initial experiments with this module more convenient."
-
-# Git add, commit, and push the newly created AMI IDs as a markdown doc to the repo
-git-add-commit-push \
-  --path "$AMI_LIST_MARKDOWN_DIR/$PACKER_BUILD_NAME-list.md" \
-  --message "$GIT_COMMIT_MESSAGE" \
-  --user-name "$GIT_USER_NAME" \
-  --user-email "$GIT_USER_EMAIL" \
-  --git-push-behavior "current" \
-  --branch-name "$BRANCH_NAME"
+if [[ -z "$PUBLISH_AMI_AWS_ACCESS_KEY_ID" || -z "$PUBLISH_AMI_AWS_SECRET_ACCESS_KEY" ]]; then
+  echo "The PUBLISH_AMI_AWS_ACCESS_KEY_ID and PUBLISH_AMI_AWS_SECRET_ACCESS_KEY environment variables must be set to the AWS credentials to use to publish the AMIs."
+  exit 1
+fi
+
+readonly packer_template_path="$1"
+readonly builder_name="$2"
+
+regions_response=$(aws ec2 describe-regions --region "us-east-1")
+all_aws_regions=$(echo "$regions_response" | jq -r '.Regions | map(.RegionName) | join(",")')
+
+echo "Building Packer template $packer_template_path (builder: $builder_name) and sharing it with all AWS accounts in the following regions: $all_aws_regions"
+
+# Copying AMIs to many regions can take longer than Packer's default wait timeouts, so we increase them here per
+# https://github.com/hashicorp/packer/issues/6536
+export AWS_MAX_ATTEMPTS=240
+export AWS_POLL_DELAY_SECONDS=15
+
+# We publish the AMIs to a different AWS account, so set those credentials
+export AWS_ACCESS_KEY_ID="$PUBLISH_AMI_AWS_ACCESS_KEY_ID"
+export AWS_SECRET_ACCESS_KEY="$PUBLISH_AMI_AWS_SECRET_ACCESS_KEY"
+
+packer build \
+  --only="$builder_name" \
+  -var copy_ami_to_regions="$all_aws_regions" \
+  -var share_ami_with_groups="all" \
+  "$packer_template_path"

circle.yml

@@ -27,13 +27,5 @@ deployment:
     tag: /v.*/
     commands:
       # If a new release is tagged in GitHub, build the AMIs and publish them to all regions.
-      - ~/$CIRCLE_PROJECT_REPONAME/_ci/publish-amis.sh "ubuntu16-ami"
-      - ~/$CIRCLE_PROJECT_REPONAME/_ci/publish-amis.sh "amazon-linux-ami"
-
-    branch: publish-amis
-    commands:
-      # We generally only want to build AMIs on new releases, but when we are setting up AMIs in a new account for the
-      # first time, we want to build the AMIs but NOT run automated tests, since those tests will fail without an existing
-      # AMI already in the AWS Account.
-      - ~/$CIRCLE_PROJECT_REPONAME/_ci/publish-amis.sh "ubuntu16-ami"
-      - ~/$CIRCLE_PROJECT_REPONAME/_ci/publish-amis.sh "amazon-linux-ami"
+      - ~/$CIRCLE_PROJECT_REPONAME/_ci/publish-amis.sh "/home/ubuntu/$CIRCLE_PROJECT_REPONAME/examples/consul-ami/consul.json" "ubuntu16-ami"
+      - ~/$CIRCLE_PROJECT_REPONAME/_ci/publish-amis.sh "/home/ubuntu/$CIRCLE_PROJECT_REPONAME/examples/consul-ami/consul.json" "amazon-linux-ami"

examples/consul-ami/consul.json

@@ -3,6 +3,8 @@
   "variables": {
     "aws_region": "us-east-1",
     "consul_version": "1.2.2",
+    "copy_ami_to_regions": "",
+    "share_ami_with_groups": "",
     "download_url": ""
   },
   "builders": [{
@@ -11,6 +13,8 @@
     "ami_description": "An Ubuntu 16.04 AMI that has Consul installed.",
     "instance_type": "t2.micro",
     "region": "{{user `aws_region`}}",
+    "ami_regions": "{{user `copy_ami_to_regions`}}",
+    "ami_groups": "{{user `share_ami_with_groups`}}",
     "type": "amazon-ebs",
     "source_ami_filter": {
       "filters": {
@@ -30,6 +34,8 @@
     "ami_description": "An Amazon Linux AMI that has Consul installed.",
     "instance_type": "t2.micro",
     "region": "{{user `aws_region`}}",
+    "ami_regions": "{{user `copy_ami_to_regions`}}",
+    "ami_groups": "{{user `share_ami_with_groups`}}",
     "type": "amazon-ebs",
     "source_ami_filter": {
       "filters": {