December Update

.env

@@ -1,14 +1,15 @@
-versionAzureCli=2.29.0
+versionAzureCli=2.30.0
 versionKubectl=1.22.1
 versionGit=1:2.30.2-1ubuntu1
-versionTflint=0.33.0
-versionTflintazrs=0.13.2
-versionVault=1.8.4
+versionTflint=0.33.1
+versionTflintazrs=0.14.0
+versionVault=1.9.0
 versionJq=1.6-2.1ubuntu1
 versionDockerCompose=1.29.2
-versionTfsec=0.58.14
+versionTfsec=0.61.3
 versionTerraformDocs=0.16.0
 versionAnsible=2.10.7-1
-versionPacker=1.7.6
-versionCheckov=2.0.489
+versionPacker=1.7.8
+versionCheckov=2.0.604
 versionMssqlTools=17.7.1.1
+extensionsAzureCli=resource-graph

.env.terraform

@@ -1,5 +1,5 @@
 0.13.7
 0.14.11
 0.15.5
-1.0.7
-1.0.9
+1.0.11
+1.1.0-rc1

.vscode/settings.json

@@ -1,13 +1,17 @@
 {
     "files.eol": "\n",
-    "terminal.integrated.defaultProfile.linux": "caf (rover on docker)",
+    "terminal.integrated.defaultProfile.linux": "caf (zsh)",
     "editor.tabSize": 2,
     "terminal.integrated.scrollback": 32000,
     "terminal.integrated.profiles.linux": {
         "caf (rover on docker)": {
             "path": "docker-compose",
             "args": ["-f", "rover_on_ssh_host.yml", "run", "-e", "ROVER_RUNNER=true", "--rm", "-w", "/tf/caf" ,"rover", "/usr/bin/zsh"],
             "overrideName": true
+        },
+        "caf (zsh)": {
+            "path": "zsh",
+            "overrideName": true
         }
     }
 }

Dockerfile

@@ -1,7 +1,7 @@
 ###########################################################
 # base tools and dependencies
 ###########################################################
-FROM ubuntu:21.04 as base
+FROM --platform=linux/amd64 ubuntu:21.04 as base
 
 SHELL ["/bin/bash", "-c"]
 
@@ -11,7 +11,6 @@ ARG versionAzureCli \
     versionVault \
     versionKubectl \
     versionTflint \
-    versionGit \
     versionJq \
     versionDockerCompose \
     versionTfsec \
@@ -21,6 +20,7 @@ ARG versionAzureCli \
     versionMssqlTools \
     versionTerraformDocs \
     versionTflintazrs \
+    extensionsAzureCli \
     SSH_PASSWD
 
 ARG USERNAME=vscode
@@ -34,7 +34,6 @@ ENV SSH_PASSWD=${SSH_PASSWD} \
     versionKubectl=${versionKubectl} \
     versionTflint=${versionTflint} \
     versionJq=${versionJq} \
-    versionGit=${versionGit} \
     versionDockerCompose=${versionDockerCompose} \
     versionTfsec=${versionTfsec} \
     versionAnsible=${versionAnsible} \
@@ -43,6 +42,7 @@ ENV SSH_PASSWD=${SSH_PASSWD} \
     versionMssqlTools=${versionMssqlTools} \
     versionTerraformDocs=${versionTerraformDocs} \
     versionTflintazrs=${versionTflintazrs} \
+    extensionsAzureCli=${extensionsAzureCli} \
     PATH="${PATH}:/opt/mssql-tools/bin:/home/vscode/.local/lib/shellspec/bin:/home/vscode/go/bin" \
     TF_DATA_DIR="/home/${USERNAME}/.terraform.cache" \
     TF_PLUGIN_CACHE_DIR="/home/${USERNAME}/.terraform.cache/plugin-cache" \
@@ -63,6 +63,7 @@ RUN apt-get update && \
     curl \
     ca-certificates \
     apt-transport-https \
+    git \
     gettext \
     software-properties-common \
     unzip \
@@ -97,17 +98,17 @@ RUN curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor >
     # Add Microsoft repository
     #
     curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list > /etc/apt/sources.list.d/msprod.list && \
-    curl https://packages.microsoft.com/config/ubuntu/21.04/prod.list >> /etc/apt/sources.list.d/msprod.list && \
+    #curl https://packages.microsoft.com/config/ubuntu/21.04/prod.list >> /etc/apt/sources.list.d/msprod.list && \
     #
     # Add Docker repository
     #
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor > /etc/apt/trusted.gpg.d/docker-archive-keyring.gpg && \
-    echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu hirsute stable" > /etc/apt/sources.list.d/docker.list && \
+    echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" > /etc/apt/sources.list.d/docker.list && \
     #
     # Add Terraform repository
     #
     curl -fsSL https://apt.releases.hashicorp.com/gpg | gpg --dearmor > /etc/apt/trusted.gpg.d/hashicorp-archive-keyring.gpg && \
-    echo "deb [arch=amd64] https://apt.releases.hashicorp.com hirsute main" > /etc/apt/sources.list.d/hashicorp.list && \
+    echo "deb [arch=amd64] https://apt.releases.hashicorp.com focal main" > /etc/apt/sources.list.d/hashicorp.list && \
     #
     # Kubernetes repo
     #
@@ -150,7 +151,7 @@ RUN echo "Installing docker-compose ${versionDockerCompose}..." && \
     tar -zxf /tmp/terraform-docs.tar.gz --directory=/usr/bin && \
     chmod +x /usr/bin/terraform-docs && \
     #
-    # Install baash completions for git
+    # Install bash completions for git
     #
     echo "Installing bash completions for git" && \
     mkdir -p /etc/bash_completion.d/ && \
@@ -179,6 +180,7 @@ RUN apt-get install -y python3-pip && \
     # Install Azure-cli
     #
     pip3 install azure-cli==${versionAzureCli}  && \
+    az config set extension.use_dynamic_install=yes_without_prompt && \
     #
     # Install checkov
     #
@@ -187,11 +189,19 @@ RUN apt-get install -y python3-pip && \
     #
     # Install pywinrm
     #
-    pip3 install pywinrm && \
+    pip3 install pywinrm
     #
     # Clean-up
     #
-    pip3 cache purge
+    #pip3 cache purge
+
+    #
+    # ################# Install Azure CLI extensions ###################
+    #
+    # Provide a comma separated list of Azure CLI extensions to add.
+    #
+RUN ext=(${extensionsAzureCli//,/ }); for i in "${ext[@]}"; do az extension add --name "$i"; done
+
     #
     # ################ Install apt packages ##################
     #
@@ -212,25 +222,13 @@ RUN echo "Installing Vault ${versionVault}..." && \
     rm /tmp/vault.zip
 
 RUN apt-get install -y --no-install-recommends \
-    docker-ce-cli
-
-RUN apt-get install -y --no-install-recommends \
-    golang
-
-RUN apt-get install -y --no-install-recommends \
-    git=${versionGit}
-
-RUN apt-get install -y --no-install-recommends \
-    ansible=${versionAnsible}
-
-RUN apt-get install -y --no-install-recommends \
-    openssh-server
-
-RUN apt-get install -y --no-install-recommends \
-    fonts-powerline
-
-RUN apt-get install -y --no-install-recommends \
-    jq=${versionJq}
+    docker-ce-cli \
+    golang \
+    git \
+    ansible \
+    openssh-server \
+    fonts-powerline \
+    jq
 
 RUN apt-get install -y --no-install-recommends \
     powershell && \
@@ -240,8 +238,8 @@ RUN apt-get install -y --no-install-recommends \
 RUN echo "Installing shellspec..." && \
     curl -fsSL https://git.io/shellspec | sh -s -- --yes
 
-RUN echo "Installing caflint..." && \
-    go install github.com/aztfmod/caflint@latest
+# RUN echo "Installing caflint..." && \
+#     go install github.com/aztfmod/caflint@latest
 
 RUN echo "Installing Tflint Ruleset ${versionTflintazrs} for Azure..." && \
     curl -sSL -o /tmp/tflint-ruleset-azurerm.zip https://github.com/terraform-linters/tflint-ruleset-azurerm/releases/download/v${versionTflintazrs}/tflint-ruleset-azurerm_linux_amd64.zip 2>&1 && \
@@ -303,7 +301,7 @@ RUN mkdir -p /tf/caf \
 
 
 
-COPY ./scripts/rover.sh ./scripts/tfstate_azurerm.sh ./scripts/functions.sh ./scripts/parse_command.sh ./scripts/banner.sh ./scripts/clone.sh ./scripts/walkthrough.sh ./scripts/sshd.sh ./scripts/backend.hcl.tf ./scripts/ci.sh ./scripts/cd.sh ./scripts/task.sh ./scripts/symphony_yaml.sh ./scripts/test_runner.sh ./
+COPY ./scripts/rover.sh ./scripts/tfstate.sh ./scripts/functions.sh ./scripts/parse_command.sh ./scripts/banner.sh ./scripts/clone.sh ./scripts/walkthrough.sh ./scripts/sshd.sh ./scripts/backend.hcl.tf ./scripts/backend.azurerm.tf ./scripts/ci.sh ./scripts/cd.sh ./scripts/task.sh ./scripts/symphony_yaml.sh ./scripts/test_runner.sh ./
 COPY ./scripts/ci_tasks/* ./ci_tasks/
 COPY ./scripts/lib/* ./lib/
 #

rover_on_ssh_host.yml

@@ -11,7 +11,7 @@
 version: '3.7'
 services:
   rover:
-    image: aztfmod/rover:1.0.4-2108.1802
+    image: aztfmod/rover:1.0.9-2111.0103
 
     user: vscode
 

scripts/backend.azurerm.tf

@@ -0,0 +1,4 @@
+terraform {
+    backend "azurerm" {
+    }
+}

scripts/functions.sh

@@ -341,8 +341,7 @@ function login_as_launchpad {
     echo ""
     echo "Getting launchpad coordinates from subscription: ${TF_VAR_tfstate_subscription_id}"
 
-    export keyvault=$(az keyvault list --subscription ${TF_VAR_tfstate_subscription_id} --query "[?tags.tfstate=='${TF_VAR_level}' && tags.environment=='${TF_VAR_environment}']" -o json | jq -r .[0].name)
-
+    keyvault=$(az graph query -q "Resources | where type == 'microsoft.keyvault/vaults' and ((tags.environment == '${TF_VAR_environment}' and tags.tfstate == '${TF_VAR_level}') or (tags.caf_environment == '${TF_VAR_environment}' and tags.caf_tfstate == '${TF_VAR_level}'))  | project name"  --query "data[0].name" -o tsv)
     echo " - keyvault_name: ${keyvault}"
 
     stg=$(az storage account show --ids ${id} -o json)
@@ -365,22 +364,6 @@ function login_as_launchpad {
 
     export TF_VAR_tfstate_key=${TF_VAR_tf_name}
 
-    if [ ${caf_command} == "landingzone" ]; then
-
-        if [ ${impersonate} = true ]; then
-            export SECRET_PREFIX=$(az keyvault secret show --subscription ${TF_VAR_tfstate_subscription_id} -n launchpad-secret-prefix --vault-name ${keyvault} -o json | jq -r .value) && echo " - Name: ${SECRET_PREFIX}"
-            echo "Set terraform provider context to Azure AD application launchpad "
-            export ARM_CLIENT_ID=$(az keyvault secret show --subscription ${TF_VAR_tfstate_subscription_id} -n ${SECRET_PREFIX}-client-id --vault-name ${keyvault} -o json | jq -r .value) && echo " - client id: ${ARM_CLIENT_ID}"
-            export ARM_CLIENT_SECRET=$(az keyvault secret show --subscription ${TF_VAR_tfstate_subscription_id} -n ${SECRET_PREFIX}-client-secret --vault-name ${keyvault} -o json | jq -r .value)
-            export ARM_TENANT_ID=$(az keyvault secret show --subscription ${TF_VAR_tfstate_subscription_id} -n ${SECRET_PREFIX}-tenant-id --vault-name ${keyvault} -o json | jq -r .value) && echo " - tenant id: ${ARM_TENANT_ID}"
-            export TF_VAR_logged_aad_app_objectId=$(az ad sp show --id ${ARM_CLIENT_ID} --query objectId -o tsv) && echo " - Set logged in aad app object id from keyvault: ${TF_VAR_logged_aad_app_objectId}"
-
-            echo "Impersonating with the azure session with the launchpad service principal to deploy the landingzone"
-            az login --service-principal -u ${ARM_CLIENT_ID} -p ${ARM_CLIENT_SECRET} --tenant ${ARM_TENANT_ID}
-        fi
-
-    fi
-
 }
 
 function deploy_landingzone {

scripts/rover.sh

@@ -7,7 +7,7 @@
 source /tf/rover/lib/logger.sh
 source /tf/rover/clone.sh
 source /tf/rover/walkthrough.sh
-source /tf/rover/tfstate_azurerm.sh
+source /tf/rover/tfstate.sh
 source /tf/rover/functions.sh
 source /tf/rover/parse_command.sh
 source /tf/rover/banner.sh
@@ -30,13 +30,14 @@ export TF_CACHE_FOLDER=${TF_DATA_DIR:=$(echo ~)}
 export ARM_SNAPSHOT=${ARM_SNAPSHOT:="true"}
 export ARM_USE_AZUREAD=${ARM_USE_AZUREAD:="true"}
 export ARM_STORAGE_USE_AZUREAD=${ARM_STORAGE_USE_AZUREAD:="true"}
-export impersonate=${impersonate:=false}
 export skip_permission_check=${skip_permission_check:=false}
 export symphony_run_all_tasks=true
 export debug_mode=${debug_mode:="false"}
 export devops=${devops:="false"}
 export log_folder_path=${log_folderpath:=~/.terraform.logs}
 export TF_IN_AUTOMATION="true" #Overriden in logger if log-severity is passed in.
+export TF_backend_type=${TF_backend_type:="azurerm"}
+export TFC_hostname=${TFC_hostname:="app.terraform.io"}
 
 unset PARAMS
 
@@ -161,7 +162,17 @@ while (( "$#" )); do
             ;;
         -tfc|--tfc)
             shift 1
-            export caf_command="tfc"
+            export TF_backend_type="tfc"
+            ;;
+        -tfc_organization|--tfc_organization)
+            export TFC_organization="${2}"
+            export TF_backend_type="tfc"
+            shift 2
+            ;;
+        -tfc_hostname|--tfc_hostname)
+            export TFC_hostname="${2}"
+            export TF_backend_type="tfc"
+            shift 2
             ;;
         -t|--tenant)
             export tenant=$(parameter_value --tenant ${2})
@@ -209,16 +220,13 @@ while (( "$#" )); do
                 export TF_VAR_level=$(parameter_value '-level' ${2})
                 shift 2
                 ;;
-        --impersonate)
-                export impersonate=true
-                shift 1
-                ;;
         -skip-permission-check)
                 export skip_permission_check=true
                 shift 1
                 ;;
         -var-folder)
                 expand_tfvars_folder $(parameter_value '-var-folder' ${2})
+                export TF_var_folder="${2}"
                 var_folder_set=true
                 shift 2
                 ;;
@@ -290,6 +298,7 @@ if [ "${caf_command}" != "walkthrough" ]; then
   echo "level (current)               : '$(echo ${TF_VAR_level})'"
   echo "environment                   : '$(echo ${TF_VAR_environment})'"
   echo "workspace                     : '$(echo ${TF_VAR_workspace})'"
+  echo "terraform backend type        : '$(echo ${TF_backend_type})'"
   echo "tfstate                       : '$(echo ${TF_VAR_tf_name})'"
   echo "tfstate subscription id       : '$(echo ${TF_VAR_tfstate_subscription_id})'"
   echo "target subscription           : '$(echo ${target_subscription_name})'"
@@ -314,3 +323,5 @@ export terraform_version=$(terraform --version | head -1 | cut -d ' ' -f 2)
 
 process_actions
 clean_up_variables
+
+exit ${RETURN_CODE}