ops(609, 847): Add Terraform for automated service provisioning

.circleci/config.yml

@@ -2,6 +2,8 @@ version: 2.1
 
 orbs:
   node: circleci/[email protected]
+  terraform: circleci/[email protected]
+  jq: circleci/[email protected]
 
 executors:
   docker-executor:
@@ -25,7 +27,7 @@ commands:
   docker-compose-check:
     steps:
       - run:
-          name: Ensure docker-compose exists,otherwise install it.
+          name: Ensure docker-compose exists, otherwise install it.
           command: ./scripts/docker-compose-check.sh
 
   # This allows us to use the orb stanza for node/install within other commands
@@ -208,6 +210,70 @@ commands:
           backend-appname: <<parameters.backend-appname>>
           frontend-appname: <<parameters.frontend-appname>>
 
+  deploy-infrastructure:
+    parameters:
+      cf-env:
+        type: string
+        default: dev
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+      cf-space:
+        type: string
+        default: tanf-dev
+      cf-org:
+        type: env_var_name
+        default: CF_ORG
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            apk update
+            apk add jq
+            apk add cloudfoundry-cli  --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-org: <<parameters.cf-org>>
+          cf-space: <<parameters.cf-space>>
+          cf-username: <<parameters.cf-username>>
+      - run:
+          name: Prepare Environment Variables
+          command: |
+            {
+              echo "env = \"<<parameters.cf-env>>\""
+              echo "cf_password = \"$<<parameters.cf-password>>\""
+              echo "cf_user = \"$<<parameters.cf-username>>\""
+              echo "cf_space_name = \"<<parameters.cf-space>>\""
+            } >> ./variables.tfvars
+
+      - run:
+          name: Export S3 Credentials
+          command: |
+            S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2)
+            {
+              echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\""
+              echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\""
+              echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\""
+            } >> ./backend_config.tfvars
+
+      - terraform/init:
+          path: ./terraform
+          backend_config_file: ./backend_config.tfvars
+      - terraform/validate:
+          path: ./terraform
+      - terraform/fmt:
+          path: ./terraform
+      - terraform/plan:
+          path: ./terraform
+          var_file: ./variables.tfvars
+      - terraform/apply:
+          path: ./terraform
+          var_file: ./variables.tfvars
+
 jobs:
   test-backend:
     executor: machine-executor
@@ -276,45 +342,75 @@ jobs:
       - store_artifacts:
           path: tdrs-frontend/reports/owasp_report.html
 
+  deploy-infrastructure-dev:
+    executor: terraform/default
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-infrastructure
+
   deploy-dev:
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
       - deploy-cloud-dot-gov
 
-  deploy-staging:
+  deploy-infrastructure-staging:
+    executor: terraform/default
     working_directory: ~/tdp-deploy
+    steps:
+      - deploy-infrastructure:
+          cf-env: staging
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
+
+  deploy-staging:
     executor: docker-executor
+    working_directory: ~/tdp-deploy
     steps:
       - deploy-cloud-dot-gov:
           backend-appname: tdp-backend-staging
           frontend-appname: tdp-frontend-staging
           cf-password: CF_PASSWORD_STAGING
-          cf-space: tanf-staging
           cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
 
   deploy-prod:
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
       - deploy-cloud-dot-gov:
           backend-appname: tdp-backend-prod
+          frontend-appname: tdp-frontend-prod
           cf-password: CF_PASSWORD_PROD
-          cf-space: tanf-prod
           cf-username: CF_USERNAME_PROD
-          frontend-appname: tdp-frontend-prod
+          cf-space: tanf-prod
 
 workflows:
   build-and-test:
     jobs:
+      - deploy-infrastructure-dev
       - test-backend
       - test-frontend
       - deploy-dev:
           requires:
-            - test-backend
-            - test-frontend
+            - deploy-infrastructure-dev
           filters:
             branches:
               only:
                 - raft-review
+
+
+      - deploy-infrastructure-staging:
+          filters:
+            branches:
+              only:
+                - raft-tdp-main
+
+      - deploy-staging:
+          requires:
+            - deploy-infrastructure-staging
+          filters:
+            branches:
+              only:
                 - raft-tdp-main

.gitignore

@@ -92,3 +92,10 @@ tdrs-backend/htmlcov/*
 
 # VIM
 *.swp
+
+# Terraform
+.terraform
+tfapply
+*.tfvars
+*.lock.hcl
+*.tfstate

deploy-cloudgov-docker.sh

@@ -47,7 +47,7 @@ service_exists()
 }
 
 
-# Performs a normal deployment unless rolling is specified in the fucntion call
+# Performs a normal deployment unless rolling is specified in the function call
 update_frontend()
 {
 	if [ "$1" = "rolling" ] ; then
@@ -61,7 +61,7 @@ update_frontend()
 	cf map-route $CGHOSTNAME_FRONTEND app.cloud.gov --hostname "${CGHOSTNAME_FRONTEND}"
 }
 
-# Performs a normal deployment unless rolling is specified in the fucntion call
+# Performs a normal deployment unless rolling is specified in the function call
 update_backend()
 {
 	if [ "$1" = "rolling" ] ; then
@@ -166,4 +166,4 @@ generate_jwt_cert()
 echo
 echo
 echo "to log into the site, you will want to go to https://${CGHOSTNAME_FRONTEND}.app.cloud.gov/"
-echo 'Have fun!'
+echo 'Have fun!'

scripts/cf-check.sh

@@ -1,4 +1,4 @@
- #!/bin/sh
+#!/bin/sh
 set -e
 if command -v cf /dev/null 2>&1; then
 echo The command cf is available
@@ -9,4 +9,4 @@ wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key |
 echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
 apt-get update
 apt-get install cf7-cli
-fi 
+fi 

terraform/README.md

@@ -0,0 +1,112 @@
+# Terraform
+
+These docs are verbose because this is technology with which developers will rarely interact. I suggest you settle in for a nice long read with your favorite drink of choice.
+
+## Prior Art
+
+### Persistent vs Ephemeral Infrastructure
+Adapted from: <https://github.com/HHS/Head-Start-TTADP#persistent-vs-ephemeral-infrastructure>
+
+**The infrastructure used to run this application can be categorized into two distinct types: _ephemeral_ and _persistent_**
+
+* **Ephemeral infrastructure** is all the infrastructure that is recreated each time the application deploys. Ephemeral infrastructure includes the "application(s)" (as defined in Cloud.gov), the EC2 instances the application runs on, and the routes that application utilizes. Our CircleCI configuration describes this infrastructure and deploys it to Cloud.gov.
+* **Persistent infrastructure** is all the infrastructure that remains constant and unchanged despite application deployments. Persistent infrastructure includes the database used in each development environment. Our Terraform configuration files describe this infrastructure and instantiates it on Cloud.gov.
+
+> This concept is often referred to as _mutable_ vs _immutable_ infrastructure.
+
+### Infrastructure as Code
+
+A high-level configuration syntax, called [Terraform language][language], describes our infrastructure. This allows a blueprint of our system to be versioned and treated as we do any other code. This configuration can be acted on locally by a developer if deployments need to be created manually, but it is mostly and ideally executed by CirclCI.
+
+###  Terraform workflow
+
+Terraform integrates into our CircleCI pipeline via the [Terraform orb][orb], and is formally described in our `deploy-infrastructure` CircleCI job. Upon validating its configuration, Terraform reads the current state of any already-existing remote objects to make sure that the Terraform state is up-to-date, and compares the current configuration to the prior state, noting all differences. Terraform creates a "plan" and proposes a set of change actions that should, if applied, make the remote objects match the configuration – this is the essence of _infrastructure as code_.
+
+### Terraform state
+
+We use an S3 bucket created by Cloud Foundry in Cloud.gov as our remote backend for Terraform. This backend maintains the "state" of Terraform and makes it possible for us to make automated deployments based on changes to the Terraform configuration files. **This is the only part of our infrastructure that must be manually configured.**
+
+## Local Set Up For Manual Deployments
+
+Sometimes a developer will need to run Terraform locally to perform manual operations. Perhaps a new TF State S3 bucket needs to be created in another environment, or there are new services or other major configuration changes that need to be tested first.
+
+1. **Install terraform**
+
+    - On macOS: `brew install terraform`
+    - On other platforms: [Download and install terraform][tf]
+
+1. **Install Cloud Foundry CLI tool**
+
+    - On macOS: `brew install cloudfoundry/tap/cf-cli`
+    - On other platforms: [Download and install cf][cf-install]
+
+1. **Login to Cloud Foundry**
+    ```bash
+       # login
+       cf login -a api.fr.cloud.gov --sso
+       # Follow temporary authorization code prompt.
+
+       # Select the target org (probably hhs-acf-prototyping), 
+       # and the space within which you want to build infrastructure.
+
+       # Spaces:
+       # dev = tanf-dev
+       # staging = tanf-staging
+       # prod = tanf-prod
+   ```
+
+1. **Set up Terraform environment variables**
+
+   In the `/terraform` directory, you can run the `create_tf_vars.sh` script which can be modified with details of your current environment, and will yield a `variables.tfvars` file which must be later passed in to Terraform. For more on this, check out [terraform variable definitions][tf-vars].
+
+   ```bash
+   ./create_tf_vars.sh
+
+   # Should generate a file `variables.tfvars` in the current directory.
+   # Your file should look something like this:
+   #
+   # env = "dev"
+   # cf_user = "some-dev-user"
+   # cf_password = "some-dev-password"
+   # cf_space_name = "tanf-dev"
+   #
+   ```
+### Terraform State S3 Bucket
+
+The service key details provide you with the credentials that are used with common file transfer programs by humans or configured in external systems. Typically, you would create a unique service key for each external client of the bucket to make it easy to rotate credentials in case they are leaked.
+
+1. **Create S3 Bucket for Terraform State**
+
+   ```bash
+    cf create-service s3 basic-sandbox tdp-tf-states
+   ```
+
+1. **Create service key**
+
+   ```bash
+   cf create-service-key tdp-tf-states tdp-tf-key
+   ```
+
+   > To later revoke access (e.g. when no longer required, or when compromised), you can run `cf delete-service-key tdp-tf-states tdp-tf-key`.
+
+1. **Get the credentials from the service key**
+   ```bash
+   cf service-key tdp-tf-states tdp-tf-key
+   ```
+   > **Rotating credentials:**
+   > 
+   > The S3 service creates unique IAM credentials for each application binding or service key. To rotate credentials associated with an application binding, unbind and rebind the service instance to the application. To rotate credentials associated with a service key, delete and recreate the service key.
+   
+
+<!-- Links -->
+
+[aws-config]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-config
+[aws-install]: https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html
+[cloudgov-bind]: https://cloud.gov/docs/deployment/managed-services/#bind-the-service-instance
+[cloudgov-deployer]: https://cloud.gov/docs/services/cloud-gov-service-account/
+[cloudgov-service-keys]: https://cloud.gov/docs/services/s3/#interacting-with-your-s3-bucket-from-outside-cloudgov
+[cf-install]: https://docs.cloudfoundry.org/cf-cli/install-go-cli.html
+[tf]: https://www.terraform.io/downloads.html
+[tf-vars]: https://www.terraform.io/docs/configuration/variables.html#variable-definitions-tfvars-files
+[orb]: https://circleci.com/developer/orbs/orb/circleci/terraform
+[language]: https://www.terraform.io/docs/language/index.html

terraform/create_tf_vars.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+KEYS_JSON=$(cf service-key tanf-keys deployer | grep -A4 "{")
+if [ -z "$KEYS_JSON" ]; then
+  echo "Unable to get service-keys, you may need to login to Cloud.gov first"
+  echo "Run cf login --sso and attempt to retry running this script"
+  exit 1
+fi
+
+# Requires installation of jq - https://stedolan.github.io/jq/download/
+CF_USERNAME_DEV=$(echo "$KEYS_JSON" | jq -r '.username')
+CF_PASSWORD_DEV=$(echo "$KEYS_JSON" | jq -r '.password')
+
+CF_SPACE="tanf-dev"
+CF_ENV="dev"
+
+touch variables.tfvars
+{
+	echo  "env = \"$CF_ENV\""
+	echo "cf_password = \"$CF_PASSWORD_DEV\""
+	echo "cf_user = \"$CF_USERNAME_DEV\""
+	echo "cf_space_name = \"$CF_SPACE\""
+} >> variables.tfvars