Merge pull request #9798 from hashicorp/e2e-terraform-tweaks-20200113

This PR makes two ergonomics changes, meant to get e2e builds more reproducible and ease changes.

### AMI Management

First, we pin the server AMIs to the commits associated with the build.  No more using the latest AMI a developer build in a test branch, or accidentally using a stale AMI because we forgot to build one!  Packer is to tag the AMI images with the commit sha used to generate the image, and then Terraform would look up only the AMIs associated with that sha. To minimize churn, we use the SHA associated with the latest Packer configurations, rather than SHA of all.

This has few benefits: reproducibility and avoiding accidental AMI changes and contamination of changes across branches. Also, the change is a stepping stone to an e2e pipeline that builds new AMIs automatically if Packer files changed.

The downside is that new AMIs will be generated even for irrelevant changes (e.g. spelling, commits), but I suspect that's OK. Also, an engineer will be forced to build the AMI whenever they change Packer files while iterating on e2e scripts; this hasn't been an issue for me yet, and I'll be open for iterating on that later if it proves to be an issue.

### Config Files and Packer

Second, this PR moves e2e config hcl management to Terraform instead of Packer. Currently, the config files live in `./terraform/config`, but they are baked into the servers by Packer and changes are ignored.  This current behavior surprised me, as I spent a bit of time debugging why my config changes weren't applied.  Having Terraform manage them would ease engineer's iteration.  Also, make Packer management more consistent (Packer only works `e2e/terraform/packer`), and easing the logic for AMI change detection.

The config directory is very small (100KB), and having it as an upload step adds negligible time to `terraform apply`.

e2e/terraform/README.md

@@ -73,18 +73,18 @@ If you want to bootstrap Nomad ACLs, include `-var 'nomad_acls=true'`.
 
 The `profile` field selects from a set of configuration files for Nomad,
 Consul, and Vault by uploading the files found in `./config/<profile>`. The
-profiles are as follows:
+standard profiles are as follows:
 
 * `full-cluster`: This profile is used for nightly E2E testing. It assumes at
   least 3 servers and includes a unique config for each Nomad client.
 * `dev-cluster`: This profile is used for developer testing of a more limited
   set of clients. It assumes at least 3 servers but uses the one config for
   all the Linux Nomad clients and one config for all the Windows Nomad
   clients.
-* `custom`: This profile is used for one-off developer testing of more complex
-  interactions between features. You can build your own custom profile by
-  writing config files to the `./config/custom` directory, which are protected
-  by `.gitignore`
+
+You may create additional profiles for testing more complex interactions between features.
+You can build your own custom profile by writing config files to the
+`./config/<custom name>` directory.
 
 For each profile, application (Nomad, Consul, Vault), and agent type
 (`server`, `client_linux`, or `client_windows`), the agent gets the following

e2e/terraform/compute.tf

@@ -1,3 +1,7 @@
+locals {
+  ami_prefix = "nomad-e2e-v2"
+}
+
 resource "aws_instance" "server" {
   ami                    = data.aws_ami.ubuntu_bionic_amd64.image_id
   instance_type          = var.instance_type
@@ -54,19 +58,33 @@ resource "aws_instance" "client_windows_2016_amd64" {
   }
 }
 
+data "external" "packer_sha" {
+  program = ["/bin/sh", "-c", <<EOT
+sha=$(git log -n 1 --pretty=format:%H packer)
+echo "{\"sha\":\"$${sha}\"}"
+EOT
+]
+
+}
+
 data "aws_ami" "ubuntu_bionic_amd64" {
   most_recent = true
   owners      = ["self"]
 
   filter {
     name   = "name"
-    values = ["nomad-e2e-ubuntu-bionic-amd64-*"]
+    values = ["${local.ami_prefix}-ubuntu-bionic-amd64-*"]
   }
 
   filter {
     name   = "tag:OS"
     values = ["Ubuntu"]
   }
+
+  filter {
+    name   = "tag:BuilderSha"
+    values = [data.external.packer_sha.result["sha"]]
+  }
 }
 
 data "aws_ami" "windows_2016_amd64" {
@@ -75,11 +93,16 @@ data "aws_ami" "windows_2016_amd64" {
 
   filter {
     name   = "name"
-    values = ["nomad-e2e-windows-2016-amd64-*"]
+    values = ["${local.ami_prefix}-windows-2016-amd64-*"]
   }
 
   filter {
     name   = "tag:OS"
     values = ["Windows2016"]
   }
+
+  filter {
+    name   = "tag:BuilderSha"
+    values = [data.external.packer_sha.result["sha"]]
+  }
 }

e2e/terraform/packer/README.md

@@ -17,10 +17,10 @@ $ packer --version
 1.6.4
 
 # build Ubuntu Bionic AMI
-$ packer build ubuntu-bionic-amd64.pkr.hcl
+$ ./build ubuntu-bionic-amd64
 
 # build Windows AMI
-$ packer build windows-2016-amd64.pkr.hcl
+$ ./build windows-2016-amd64
 ```
 
 ## Debugging Packer Builds

e2e/terraform/packer/build

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -e
+
+usage() {
+    cat <<EOF
+Usage: build <target>
+
+Build an AMI for the target configuration
+
+Examples
+  build ubuntu-bionic-amd64
+
+EOF
+
+    exit 2
+}
+
+if [[ $# -ne 1 ]]; then
+     usage
+fi
+
+target=${1/%.pkr.hcl/}
+
+directory=$(dirname "$0")
+cd ${directory}
+
+if ! test -f "${target}.pkr.hcl"; then
+    echo "${target}.pkr.hcl is not present" >&2
+    exit 1
+fi
+
+sha=$(git log -n 1 --pretty=format:%H ${dirname})
+echo packer build --var "build_sha=${sha}" ${target}.pkr.hcl
+packer build --var "build_sha=${sha}" ${target}.pkr.hcl

e2e/terraform/packer/ubuntu-bionic-amd64.pkr.hcl

@@ -1,10 +1,16 @@
+variable "build_sha" {
+  type        = string
+  description = "the revision of the packer scripts building this image"
+}
+
 locals {
   timestamp = regex_replace(timestamp(), "[- TZ:]", "")
   distro    = "ubuntu-bionic-18.04-amd64-server-*"
+  version   = "v2"
 }
 
 source "amazon-ebs" "latest_ubuntu_bionic" {
-  ami_name             = "nomad-e2e-ubuntu-bionic-amd64-${local.timestamp}"
+  ami_name             = "nomad-e2e-${local.version}-ubuntu-bionic-amd64-${local.timestamp}"
   iam_instance_profile = "packer_build" // defined in nomad-e2e repo
   instance_type        = "t2.medium"
   region               = "us-east-1"
@@ -23,8 +29,9 @@ source "amazon-ebs" "latest_ubuntu_bionic" {
   }
 
   tags = {
-    OS      = "Ubuntu"
-    Version = "Bionic"
+    OS         = "Ubuntu"
+    Version    = "Bionic"
+    BuilderSha = var.build_sha
   }
 }
 
@@ -36,11 +43,6 @@ build {
     source      = "./ubuntu-bionic-amd64"
   }
 
-  provisioner "file" {
-    destination = "/tmp/config"
-    source      = "../config"
-  }
-
   // cloud-init modifies the apt sources, so we need to wait
   // before running our setup
   provisioner "shell-local" {

e2e/terraform/packer/ubuntu-bionic-amd64/setup.sh

@@ -77,7 +77,6 @@ mkdir_for_root $NOMAD_PLUGIN_DIR
 sudo mv /tmp/linux/nomad.service /etc/systemd/system/nomad.service
 
 echo "Install Nomad"
-sudo mv /tmp/config /opt/
 sudo mv /tmp/linux/provision.sh /opt/provision.sh
 sudo chmod +x /opt/provision.sh
 /opt/provision.sh --nomad_version $NOMADVERSION --nostart

e2e/terraform/packer/windows-2016-amd64.pkr.hcl

@@ -1,7 +1,15 @@
-locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
+variable "build_sha" {
+  type        = string
+  description = "the revision of the packer scripts building this image"
+}
+
+locals {
+  timestamp = regex_replace(timestamp(), "[- TZ:]", "")
+  version   = "v2"
+}
 
 source "amazon-ebs" "latest_windows_2016" {
-  ami_name       = "nomad-e2e-windows-2016-amd64-${local.timestamp}"
+  ami_name       = "nomad-e2e-${local.version}-windows-2016-amd64-${local.timestamp}"
   communicator   = "ssh"
   instance_type  = "t2.medium"
   region         = "us-east-1"
@@ -20,7 +28,8 @@ source "amazon-ebs" "latest_windows_2016" {
   }
 
   tags = {
-    OS = "Windows2016"
+    OS         = "Windows2016"
+    BuilderSha = var.build_sha
   }
 }
 
@@ -37,11 +46,6 @@ build {
     ]
   }
 
-  provisioner "file" {
-    destination = "/opt"
-    source      = "../config"
-  }
-
   provisioner "file" {
     destination = "/opt/provision.ps1"
     source      = "./windows-2016-amd64/provision.ps1"

e2e/terraform/provision-nomad/main.tf

@@ -1,28 +1,22 @@
 locals {
   provision_script = var.platform == "windows_amd64" ? "C:/opt/provision.ps1" : "/opt/provision.sh"
 
-  custom_path = dirname("${path.root}/config/custom/")
-
-  custom_config_files = compact(setunion(
-    fileset(local.custom_path, "nomad/*.hcl"),
-    fileset(local.custom_path, "nomad/${var.role}/*.hcl"),
-    fileset(local.custom_path, "nomad/${var.role}/indexed/*${var.index}.hcl"),
-    fileset(local.custom_path, "consul/*.json"),
-    fileset(local.custom_path, "consul/${var.role}/*.json"),
-    fileset(local.custom_path, "consul${var.role}indexed/*${var.index}*.json"),
-    fileset(local.custom_path, "vault/*.hcl"),
-    fileset(local.custom_path, "vault${var.role}*.hcl"),
-    fileset(local.custom_path, "vault${var.role}indexed/*${var.index}.hcl"),
+  config_path = dirname("${path.root}/config/")
+
+  config_files = compact(setunion(
+    fileset(local.config_path, "**"),
   ))
 
+  update_config_command = var.platform == "windows_amd64" ? "if (test-path /opt/config) { Remove-Item -Path /opt/config -Force -Recurse }; cp -r /tmp/config /opt/config" : "sudo rm -rf /opt/config; sudo mv /tmp/config /opt/config"
+
   # abstract-away platform-specific parameter expectations
   _arg = var.platform == "windows_amd64" ? "-" : "--"
 }
 
 resource "null_resource" "provision_nomad" {
 
   depends_on = [
-    null_resource.upload_custom_configs,
+    null_resource.upload_configs,
     null_resource.upload_nomad_binary
   ]
 
@@ -85,7 +79,7 @@ data "template_file" "arg_index" {
 resource "null_resource" "upload_nomad_binary" {
 
   count      = var.nomad_local_binary != "" ? 1 : 0
-  depends_on = [null_resource.upload_custom_configs]
+  depends_on = [null_resource.upload_configs]
   triggers = {
     nomad_binary_sha = filemd5(var.nomad_local_binary)
   }
@@ -105,11 +99,10 @@ resource "null_resource" "upload_nomad_binary" {
   }
 }
 
-resource "null_resource" "upload_custom_configs" {
+resource "null_resource" "upload_configs" {
 
-  count = var.profile == "custom" ? 1 : 0
   triggers = {
-    hashes = join(",", [for file in local.custom_config_files : filemd5("${local.custom_path}/${file}")])
+    hashes = join(",", [for file in local.config_files : filemd5("${local.config_path}/${file}")])
   }
 
   connection {
@@ -122,7 +115,12 @@ resource "null_resource" "upload_custom_configs" {
   }
 
   provisioner "file" {
-    source      = local.custom_path
+    source      = local.config_path
     destination = "/tmp/"
   }
+
+  provisioner "local-exec" {
+    command = "until ssh -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${var.connection.private_key} -p ${var.connection.port} ${var.connection.user}@${var.connection.host} '${local.update_config_command}'; do sleep 5; done"
+  }
+
 }