From dc036ef6265fa2fd9bf485f318179e5bad6da405 Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Fri, 11 Mar 2022 14:18:14 +0000 Subject: [PATCH 001/142] Initial commit --- Makefile | 4 + .../gitea/terraform/.terraform.lock.hcl | 27 +-- .../shared_services/nexus-cert/.dockerignore | 4 + .../shared_services/nexus-cert/.gitignore | 2 + .../nexus-cert/Dockerfile.tmpl | 21 +++ .../shared_services/nexus-cert/azure.json | 32 ++++ .../nexus-cert/parameters.json | 38 ++++ .../shared_services/nexus-cert/porter.yaml | 80 ++++++++ .../nexus-cert/scripts/letsencrypt.sh | 94 ++++++++++ .../nexus-cert/terraform/appgateway.tf | 171 ++++++++++++++++++ .../nexus-cert/terraform/certificate.tf | 49 +++++ .../nexus-cert/terraform/locals.tf | 25 +++ .../nexus-cert/terraform/main.tf | 15 ++ .../nexus-cert/terraform/output.tf | 11 ++ .../nexus-cert/terraform/staticweb.tf | 30 +++ .../nexus-cert/terraform/variables.tf | 3 + 16 files changed, 593 insertions(+), 13 deletions(-) create mode 100644 templates/shared_services/nexus-cert/.dockerignore create mode 100644 templates/shared_services/nexus-cert/.gitignore create mode 100644 templates/shared_services/nexus-cert/Dockerfile.tmpl create mode 100644 templates/shared_services/nexus-cert/azure.json create mode 100644 templates/shared_services/nexus-cert/parameters.json create mode 100644 templates/shared_services/nexus-cert/porter.yaml create mode 100644 templates/shared_services/nexus-cert/scripts/letsencrypt.sh create mode 100644 templates/shared_services/nexus-cert/terraform/appgateway.tf create mode 100644 templates/shared_services/nexus-cert/terraform/certificate.tf create mode 100644 templates/shared_services/nexus-cert/terraform/locals.tf create mode 100644 templates/shared_services/nexus-cert/terraform/main.tf create mode 100644 templates/shared_services/nexus-cert/terraform/output.tf create mode 100644 templates/shared_services/nexus-cert/terraform/staticweb.tf create mode 100644 templates/shared_services/nexus-cert/terraform/variables.tf diff --git a/Makefile b/Makefile index f0a03071c5..0629911037 100644 --- a/Makefile +++ b/Makefile @@ -137,6 +137,10 @@ nexus-install: $(call target_title, "Installing Nexus") \ && make SHARED_SERVICE_KEY=shared-service-sonatype-nexus TF_VAR_nexus_properties_path=../nexus.properties terraform-shared-service-deploy DIR=./templates/shared_services/sonatype-nexus/terraform +nexus-cert-install: + $(call target_title, "Installing Nexus Cert") \ + && make SHARED_SERVICE_KEY=shared-service-nexus-cert terraform-shared-service-deploy DIR=./templates/shared_services/nexus-cert/terraform + # / End migration targets deploy-core: tre-start diff --git a/templates/shared_services/gitea/terraform/.terraform.lock.hcl b/templates/shared_services/gitea/terraform/.terraform.lock.hcl index 00b91198f9..fe444a7bf4 100644 --- a/templates/shared_services/gitea/terraform/.terraform.lock.hcl +++ b/templates/shared_services/gitea/terraform/.terraform.lock.hcl @@ -21,20 +21,21 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.1.0" + version = "2.2.1" hashes = [ - "h1:EYZdckuGU3n6APs97nS2LxZm3dDtGqyM4qaIvsmac8o=", - "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", - "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", - "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", - "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", - "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", - "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", - "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", - "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", - "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", - "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", - "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", + "h1:y1SV7/L0B2/q30waki+emxW2+e8+fyEv+m53nwH6ME4=", + "zh:15282174d8e0644a86c21c25d4ea1eaff2950fffc5eeb0281cbebd74c13cfd06", + "zh:46cd90f69cfd7dad613dc71606e25c339cdaabe8d5ebc1ad712c4c30747ec0fd", + "zh:518519a26a709b7a8ca2f9282389d2ac08eab057e492805655970e7eca25cab8", + "zh:5ba89d886ada09aee3926d8340853d0bf88953f3ede2cf6d7af3f2a41fec642e", + "zh:6a20f6ca3a24af94c88a5b9afa50d4fd7d47c39ee8c0184c415aaae14204c497", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a3f65fda390fcbfa7368f8d6da4d261b64383f3c86b5667e07a2176e27278cda", + "zh:a5f3ab57445b1974d3064ed223d3c4b21e89d375e46384fed65c199241003b76", + "zh:a8fae6bd2d1b233c0cf5690d68e29930d73fbf167dd61499597ed4acadf1f4a2", + "zh:b3c848292d0ca01f7da81a5a945ab939817bac032110d9a1b07d8264490db1b1", + "zh:e0fdc5cc40cb9c0726d2a195a1e48f5e43bc33f3b3f6dfbda72ca7010bc00a75", + "zh:ef6139bfa8bcbc2ed6f07237362ce347a90500722314676e1203cf6c8f8ef2ad", ] } diff --git a/templates/shared_services/nexus-cert/.dockerignore b/templates/shared_services/nexus-cert/.dockerignore new file mode 100644 index 0000000000..2919244c86 --- /dev/null +++ b/templates/shared_services/nexus-cert/.dockerignore @@ -0,0 +1,4 @@ +# See https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Put files here that you don't want copied into your bundle's invocation image +.gitignore +Dockerfile.tmpl diff --git a/templates/shared_services/nexus-cert/.gitignore b/templates/shared_services/nexus-cert/.gitignore new file mode 100644 index 0000000000..73a68e4976 --- /dev/null +++ b/templates/shared_services/nexus-cert/.gitignore @@ -0,0 +1,2 @@ +.cnab/ +.terraform* diff --git a/templates/shared_services/nexus-cert/Dockerfile.tmpl b/templates/shared_services/nexus-cert/Dockerfile.tmpl new file mode 100644 index 0000000000..9a66e15f55 --- /dev/null +++ b/templates/shared_services/nexus-cert/Dockerfile.tmpl @@ -0,0 +1,21 @@ +FROM debian:stretch-slim + +ARG BUNDLE_DIR + +RUN apt-get update && apt-get install -y ca-certificates + +# This is a template Dockerfile for the bundle's invocation image +# You can customize it to use different base images, install tools and copy configuration files. +# +# Porter will use it as a template and append lines to it for the mixins +# and to set the CMD appropriately for the CNAB specification. +# +# Add the following line to porter.yaml to instruct Porter to use this template +# dockerfile: Dockerfile.tmpl + +# You can control where the mixin's Dockerfile lines are inserted into this file by moving "# PORTER_MIXINS" line +# another location in this file. If you remove that line, the mixins generated content is appended to this file. +# PORTER_MIXINS + +# Use the BUNDLE_DIR build argument to copy files into the bundle +COPY . $BUNDLE_DIR diff --git a/templates/shared_services/nexus-cert/azure.json b/templates/shared_services/nexus-cert/azure.json new file mode 100644 index 0000000000..cdc4c1365c --- /dev/null +++ b/templates/shared_services/nexus-cert/azure.json @@ -0,0 +1,32 @@ +{ + "schemaVersion": "1.0.0-DRAFT+b6c701f", + "name": "azure", + "created": "2021-06-03T11:31:05.7314113Z", + "modified": "2021-06-03T11:31:05.7314113Z", + "credentials": [ + { + "name": "azure_client_id", + "source": { + "env": "ARM_CLIENT_ID" + } + }, + { + "name": "azure_client_secret", + "source": { + "env": "ARM_CLIENT_SECRET" + } + }, + { + "name": "azure_subscription_id", + "source": { + "env": "ARM_SUBSCRIPTION_ID" + } + }, + { + "name": "azure_tenant_id", + "source": { + "env": "ARM_TENANT_ID" + } + } + ] +} diff --git a/templates/shared_services/nexus-cert/parameters.json b/templates/shared_services/nexus-cert/parameters.json new file mode 100644 index 0000000000..7c41929a0a --- /dev/null +++ b/templates/shared_services/nexus-cert/parameters.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": "1.0.0-DRAFT", + "name": "base", + "created": "2021-06-04T13:37:29.5071039+03:00", + "modified": "2021-06-04T13:37:29.5071039+03:00", + "parameters": [ + { + "name": "tre_id", + "source": { + "env": "TRE_ID" + } + }, + { + "name": "azure_location", + "source": { + "env": "LOCATION" + } + }, + { + "name": "tfstate_container_name", + "source": { + "env": "TERRAFORM_STATE_CONTAINER_NAME" + } + }, + { + "name": "tfstate_resource_group_name", + "source": { + "env": "MGMT_RESOURCE_GROUP_NAME" + } + }, + { + "name": "tfstate_storage_account_name", + "source": { + "env": "MGMT_STORAGE_ACCOUNT_NAME" + } + } + ] +} diff --git a/templates/shared_services/nexus-cert/porter.yaml b/templates/shared_services/nexus-cert/porter.yaml new file mode 100644 index 0000000000..383380af76 --- /dev/null +++ b/templates/shared_services/nexus-cert/porter.yaml @@ -0,0 +1,80 @@ +--- +name: tre-shared-service-nexus-cert +version: 0.0.1 +description: "An Azure TRE Nexus certificate creation shared service" +registry: azuretre +dockerfile: Dockerfile.tmpl + +credentials: + - name: azure_tenant_id + env: ARM_TENANT_ID + - name: azure_subscription_id + env: ARM_SUBSCRIPTION_ID + - name: azure_client_id + env: ARM_CLIENT_ID + - name: azure_client_secret + env: ARM_CLIENT_SECRET + +parameters: + - name: tre_id + type: string + description: "The ID of the parent TRE instance e.g., mytre-dev-3142" + - name: azure_location + type: string + description: "Azure location (region) to deploy to" + - name: tfstate_resource_group_name + type: string + description: "Resource group containing the Terraform state storage account" + - name: tfstate_storage_account_name + type: string + description: "The name of the Terraform state storage account" + - name: tfstate_container_name + type: string + default: "tfstate" + description: "The name of the Terraform state storage container" + +mixins: + - exec + - terraform: + clientVersion: 1.1.5 + +install: + - terraform: + description: "Deploy shared service" + input: false + vars: + tre_id: "{{ bundle.parameters.tre_id }}" + location: "{{ bundle.parameters.azure_location }}" + backendConfig: + resource_group_name: + "{{ bundle.parameters.tfstate_resource_group_name }}" + storage_account_name: + "{{ bundle.parameters.tfstate_storage_account_name }}" + container_name: + "{{ bundle.parameters.tfstate_container_name }}" + key: + "{{ bundle.parameters.tre_id }}-shared-service-firewall" + +upgrade: + - exec: + description: "Upgrade shared service" + command: echo + arguments: + - "This shared service does not implement upgrade action" + +uninstall: + - terraform: + description: "Tear down shared service" + input: false + vars: + tre_id: "{{ bundle.parameters.tre_id }}" + location: "{{ bundle.parameters.azure_location }}" + backendConfig: + resource_group_name: + "{{ bundle.parameters.tfstate_resource_group_name }}" + storage_account_name: + "{{ bundle.parameters.tfstate_storage_account_name }}" + container_name: + "{{ bundle.parameters.tfstate_container_name }}" + key: + "{{ bundle.parameters.tre_id }}-shared-service-firewall" diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh new file mode 100644 index 0000000000..d2bcddd26b --- /dev/null +++ b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh @@ -0,0 +1,94 @@ +#!/bin/bash +set -e + +script_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")) + +if [[ -z ${STORAGE_ACCOUNT} ]]; then + echo "STORAGE_ACCOUNT not set" + exit 1 +fi + + +echo "Checking for index.html file in storage account" + +# Create the default index.html page +cat << EOF > index.html + + +EOF + +indexExists=$(az storage blob list -o json \ + --account-name "${STORAGE_ACCOUNT}" \ + --auth-mode login \ + --container-name '$web' \ + --query "[?name=='index.html'].name" \ + | jq 'length') + +if [[ ${indexExists} -lt 1 ]]; then + echo "Uploading index.html file" + + az storage blob upload \ + --account-name "${STORAGE_ACCOUNT}" \ + --auth-mode login \ + --container-name '$web' \ + --file index.html \ + --name index.html \ + --no-progress \ + --only-show-errors + + # Wait a bit for the App Gateway health probe to notice + echo "Waiting 30s for health probe" + sleep 30s +else + echo "index.html already present" +fi + +ledir=$(pwd)/letsencrypt + +mkdir -p "${ledir}/logs" + +# Initiate the ACME challange +/opt/certbot/bin/certbot certonly \ + --config-dir ${ledir} \ + --work-dir ${ledir} \ + --logs-dir ${ledir}/logs \ + --manual \ + --preferred-challenges=http \ + --manual-auth-hook ${script_dir}/auth-hook.sh \ + --manual-cleanup-hook ${script_dir}/cleanup-hook.sh \ + --domain $FQDN \ + --non-interactive \ + --agree-tos \ + --register-unsafely-without-email + +# Convert the generated certificate to a .pfx +CERT_DIR="${ledir}/live/$FQDN" +CERT_PASSWORD=$(openssl rand -base64 30) +openssl pkcs12 -export \ + -inkey "${CERT_DIR}/privkey.pem" \ + -in "${CERT_DIR}/fullchain.pem" \ + -out "${CERT_DIR}/aci.pfx" \ + -passout "pass:${CERT_PASSWORD}" + +if [[ -n ${KEYVAULT} ]]; then + sid=$(az keyvault certificate import \ + -o json \ + --vault-name ${KEYVAULT} \ + --name 'nexus-letsencrypt' \ + --file "${CERT_DIR}/aci.pfx" \ + --password "${CERT_PASSWORD}" \ + | jq -r '.sid') + + az network application-gateway ssl-cert update \ + --resource-group "${RESOURCE_GROUP_NAME}" \ + --gateway-name "${APPLICATION_GATEWAY}" \ + --name 'cert-nexus-primary' \ + --key-vault-secret-id "${sid}" +else + az network application-gateway ssl-cert update \ + --resource-group "${RESOURCE_GROUP_NAME}" \ + --gateway-name "${APPLICATION_GATEWAY}" \ + --name 'nexus-letsencrypt' \ + --cert-file "${CERT_DIR}/aci.pfx" \ + --cert-password "${CERT_PASSWORD}" +fi diff --git a/templates/shared_services/nexus-cert/terraform/appgateway.tf b/templates/shared_services/nexus-cert/terraform/appgateway.tf new file mode 100644 index 0000000000..c23c357034 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/appgateway.tf @@ -0,0 +1,171 @@ +resource "azurerm_public_ip" "appgwpip" { + name = "pip-nexus-${var.tre_id}" + resource_group_name = local.core_resource_group_name + location = var.location + allocation_method = "Static" + sku = "Standard" + domain_name_label = "nexus-${var.tre_id}" + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_user_assigned_identity" "agw_id" { + resource_group_name = local.core_resource_group_name + location = var.location + name = "id-agw-nexuscert-${var.tre_id}" + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_application_gateway" "agw" { + name = "agw-nexuscert-${var.tre_id}" + resource_group_name = local.core_resource_group_name + location = var.location + + sku { + name = "Standard_v2" + tier = "Standard_v2" + capacity = 1 + } + + # User-assign managed identify id required to access certificate in KeyVault + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.agw_id.id] + } + + # Internal subnet for gateway backend. + gateway_ip_configuration { + name = "gateway-ip-configuration" + subnet_id = data.azurerm_subnet.app_gw_subnet.id + } + + # HTTP Port + frontend_port { + name = local.insecure_frontend_port_name + port = 80 + } + + # HTTPS Port + frontend_port { + name = local.secure_frontend_port_name + port = 443 + } + + # Public front-end + frontend_ip_configuration { + name = local.frontend_ip_configuration_name + public_ip_address_id = azurerm_public_ip.appgwpip.id + } + + # Primary SSL cert linked to KeyVault + ssl_certificate { + name = local.certificate_name + key_vault_secret_id = azurerm_key_vault_certificate.tlscert.secret_id + } + + # Backend pool with the static website in storage account. + backend_address_pool { + name = local.staticweb_backend_pool_name + fqdns = [azurerm_storage_account.staticweb.primary_web_host] + } + + # Backend settings for static web. + backend_http_settings { + name = local.staticweb_http_setting_name + cookie_based_affinity = "Disabled" + port = 443 + protocol = "Https" + request_timeout = 60 + pick_host_name_from_backend_address = true + } + + # Public HTTPS listener + http_listener { + name = local.secure_listener_name + frontend_ip_configuration_name = local.frontend_ip_configuration_name + frontend_port_name = local.secure_frontend_port_name + protocol = "Https" + ssl_certificate_name = local.certificate_name + } + + # Public HTTP listener + http_listener { + name = local.insecure_listener_name + frontend_ip_configuration_name = local.frontend_ip_configuration_name + frontend_port_name = local.insecure_frontend_port_name + protocol = "Http" + } + + request_routing_rule { + name = local.request_routing_rule_name + rule_type = "PathBasedRouting" + http_listener_name = local.secure_listener_name + url_path_map_name = local.app_path_map_name + } + + # Routing rule to redirect non-secure traffic to HTTPS + request_routing_rule { + name = local.redirect_request_routing_rule_name + rule_type = "PathBasedRouting" + http_listener_name = local.insecure_listener_name + url_path_map_name = local.redirect_path_map_name + } + + # Default traffic is routed to the static website. + url_path_map { + name = local.app_path_map_name + default_backend_address_pool_name = local.staticweb_backend_pool_name + default_backend_http_settings_name = local.staticweb_http_setting_name + + path_rule { + name = "all" + paths = ["/*"] + backend_address_pool_name = local.staticweb_backend_pool_name + backend_http_settings_name = local.staticweb_http_setting_name + } + } + + # Redirect any HTTP traffic to HTTPS unless its the ACME challenge path used for LetsEncrypt validation. + url_path_map { + name = local.redirect_path_map_name + default_redirect_configuration_name = local.redirect_configuration_name + + path_rule { + name = "acme" + paths = ["/.well-known/acme-challenge/*"] + backend_address_pool_name = local.staticweb_backend_pool_name + backend_http_settings_name = local.staticweb_http_setting_name + } + } + + # Redirect to HTTPS + redirect_configuration { + name = local.redirect_configuration_name + redirect_type = "Permanent" + target_listener_name = local.secure_listener_name + include_path = true + include_query_string = true + } + + # We don't want Terraform to revert certificate cycle changes. We assume the certificate will be renewed in keyvault. + lifecycle { + ignore_changes = [ + ssl_certificate, + tags + ] + } + +} + +data "azurerm_subnet" "app_gw_subnet" { + name = "AppGwSubnet" + virtual_network_name = "vnet-${var.tre_id}" + resource_group_name = local.core_resource_group_name +} + +data "azurerm_public_ip" "appgwpip_data" { + depends_on = [azurerm_application_gateway.agw] + name = "pip-nexus-${var.tre_id}" + resource_group_name = local.core_resource_group_name +} diff --git a/templates/shared_services/nexus-cert/terraform/certificate.tf b/templates/shared_services/nexus-cert/terraform/certificate.tf new file mode 100644 index 0000000000..fabcd7c972 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/certificate.tf @@ -0,0 +1,49 @@ +resource "azurerm_key_vault_access_policy" "app_gw_managed_identity" { + key_vault_id = data.azurerm_key_vault.key_vault.id + tenant_id = azurerm_user_assigned_identity.agw_id.tenant_id + object_id = azurerm_user_assigned_identity.agw_id.principal_id + + key_permissions = [ + "Get", + ] + + secret_permissions = [ + "Get", + ] +} + +resource "azurerm_key_vault_certificate" "tlscert" { + name = "nexus-letsencrypt" + key_vault_id = data.azurerm_key_vault.key_vault.id + + # This is a temporary self-signed cert for CN=temp + certificate { + contents = "MIIKOgIBAzCCCfYGCSqGSIb3DQEHAaCCCecEggnjMIIJ3zCCBgAGCSqGSIb3DQEHAaCCBfEEggXtMIIF6TCCBeUGCyqGSIb3DQEMCgECoIIE/jCCBPowHAYKKoZIhvcNAQwBAzAOBAiyQRLJomrNiwICB9AEggTYT6yfTxjl0APqhcP+C2qQp/NbcyJfgpSJf6EEnWEcOuxXdaipNsClUyzaJ8B+v7GCOwjkoMqY/jgO4o8d5SlZbpu+yZ3ItTWQlPbFK83uwn8kTxZzhScEujsV2txPNxe5z/kyrrgNyvRlOcmasBhN24hrv0SkuGJHu31xc+5I7AJjSbblp8ccDQgJHjHAf1oMxz05ZhQazfRZU6WElDvvMP5ZGybxGpyksw4/A9GZz0XiMiJha3bJqpyAeZ/HXYucgnO5/ztOnCNgQC+qaQdOxSKLub0aFwfnnRZtfdRE4JcFRMr4ONGaUOR59OB6MYBZmk+X1sPFar/f+zNA4j6/yhTX6gf0EbjPjmDu1WP02qiJ9jTZv0pD07J98eHU58iir7i/E5eyJ/8VYpinBJC4OCwyGyDWysv1XY7TnZmT9Vwu27I/1ONseY1wbwXarqixXJVhUXLHsMwluNAGUNigMZqMm+3UEsam0Q2ie5FPCWusLBaASvzNL8t9oyLEhJ48Op/7fcX3/OXb8EIceEpNCBZD0nRLsxWA03aO/+vp4iGKhxlzsTx7YqLKFU7ADxsZChYgzLIHAVEfrxPIRWYUIqzbSWutCswMdvw2iZRfWHDpz1jvNJ0aErRyDCQRimjm8kLTzaDTuAscmefmG1cx465I7olRpeVv96i3tBbf0xrKcqjQuz4yepAzvFcPv48GTrM6thngUdjApcPgZScsAv71V2WvW3mQF/TywJlvOSdsG5Hr6bH6jD6PEBkgwBtjZpKokzoWNgw3qpeqiGRdcpYQRv4qrad70Lk99wcaM1wQ5AfZDUTx+PWVlHL/+ikTO+i0UgXoQHp9JFG9vhRdT82tgNKEcFXWWng1+Vlpqk3l5XLi1MZWmBGMPNRb4uqVgH4ekWEvdZEyClbrIGOUe6zPntMsrDXh0M+zi9jqVrB88gLXgTw1UweGwR/xOlRZa/bBg4RwDzcePABia3A4ksREIatZ5rJFipTc6edeaQWFT6V5Q7nqv1Zzoxp3qV7zhc9eCkReOQCTs8Ds8uALP/szLIdT7vPIslme6PTX1mzjBWMnDhoca8xGvEPGBMgxY9kozQXwlgp2JdXw5JuW99rEBkpGuEtDu+yyjiBz80EKIi7kN8VltxAfZsOOt5KpHGleAdkhQifO8c92RC4oivzOREUKtD9cusqjdMrbXHWaJ9mBKbwdvR3SqqGGgxKC3I0nX7jYbHAC7HZTkq7Nuh5QcQmqjaAuOozLUiiv3eVO3FGMmSNFkXtf96NAE+66lx6q8PSXuOJl/EGNS94d5xNyLupqLljFgAwtPRtX/V01BqfuKgwqVAfwP2knBHhJ/CUi0wdG6Ev8YTPnQ+3OG2b0NGM7le6Qi9j07qgpWO2s0xS0jt3FGJeJi/LKxHp7A/7asKp5Evzrw+Nejfbit3EL8fml8DjLDZS7D/6HWYiVEqW85IDf2+iWOGztZIczeHYQqX7t7POv3G22x+PyXHCPSOF4F40sk0Mzw3FaWyow7KDpc0p3u1h/GuLKXgiP0ixNcmCCBdmHfYF/uCiaVoQGeh1rg08tnATdDOozBwaSB1jhiMn3DADy/7kGcej3Da+7elnY4ivWzWZEtv465eDrruxsalcTdbitNOUiPJM/Mos/bjwvlDGB0zATBgkqhkiG9w0BCRUxBgQEAQAAADBdBgkqhkiG9w0BCRQxUB5OAHQAZQAtADAAOQA4ADIANAA5AGUAZQAtADkAZgAxADEALQA0AGUAZQBjAC0AYQAxADYAMwAtAGMAZgBiAGMAZgBhADMANwA5ADQANQBiMF0GCSsGAQQBgjcRATFQHk4ATQBpAGMAcgBvAHMAbwBmAHQAIABTAG8AZgB0AHcAYQByAGUAIABLAGUAeQAgAFMAdABvAHIAYQBnAGUAIABQAHIAbwB2AGkAZABlAHIwggPXBgkqhkiG9w0BBwagggPIMIIDxAIBADCCA70GCSqGSIb3DQEHATAcBgoqhkiG9w0BDAEDMA4ECM9JyfoDC5veAgIH0ICCA5BHYKLRmWNcYkQVs7Hn/BAw9uTJ+kzH9ZkjwgoRl0aSanS0TIOH+gKwjJgy4YIenm2H1KXoNhDBqqxXDvBIF1K5JP7OsQMCiViU8pdL8GmC5NOE4SPokIRe+2fUQBHtzaZx4GtkwpHq1Tlsv+2U9EmeuQTBX2C1Amj9Dup2uwYrGfSx6bVoIP8yEzB29f/3gxSmNLQAvlXjO5hdYZM6dDRX+SaYmIJHPg8SYG6EuzaJ2a++f2qJ1f7XtgB/Yt9/KPP4AnPTaEQMyR08CyJTf2MxAOBHHza+x4SqaWOEzL/SeOTalaQTY5vj7c6LkgYIXiyYi1QK3VHeJ+NYPBFgu4+B0Sk1ef8jZ6XeSnsxXj6lhLb8sUiemwsEfueh3RKpOhzQEZDRVYP6B/k3Nv9JwuOUiSWdgPGkhyTWu+xOD4+bixCMUJ0AI+MLEjI+ixkP429Ylp9nxyWxPUTG+PW2AR/euuCW222yEjzJzz0VPJXWeUWVyRzoYicrBWEvK3/KnnkWafYti2Ugsee5gCMCmeKeFKoCKEwdMzdAfafC3FI6x2LnUGNefv6E3hIzw6v0vYlEUmmgb5aI/zjCCXix4lSuqYtvfwtLZIf4aEu1OoCRE35zkHlIUh2Cf5kERPawNM6V/qnKEV/SsQQyNtOI2iUi11cBHkGwOxDzBiWkidhOZ8UB9r+453G8zWNWwXWdhxoYHF1CYDtjy2lrzw+ou5jW0yI1G82a2UpZkPMtMoXTuoErVgAfW2jZ894vFEsIqeny1FVoQtpKhmdX1hatxR3AyRYR2L9tdD1BcGMll6FkgJjqYNcV8mbZzLzFtYNoSljGGOr46xhW4Bfv1jPw2eKXJULMWwWxaHq4rTSbWq5q5eElM0OkPmAyrqoVbd8uwATmIqtwOgmJqeIh0HgmkuaJfDMSfOTBnamS2B9Fl01TvNV5EWmMC1qROBW8yi/7meEPmDyfWOHz5v6ltG5Ra5xC0UHv/ljYpce1kWKPTlGUEknhZwAbeyDpuy6ljneGtSdQPz9uqrzyMnDWGEguVQYubmxVbXICwzGI06UBB9JVRuXPuUHj3j0rSLny/9/A1sUb2kwXNhVlOQGMDWW/kcY5SsWqTb+z+rXL90aaaWmc6Mu0tJv/HZMraJ3KDR6Y1WXUUzYHBegMtwOWTtyVwb0b8W2Oo0c0g3t1dNcVY/nYbikx/lgC4Xanco7+i4gCVA0wOzAfMAcGBSsOAwIaBBR2TY2pU6UZFCDZUtZIO6qHyC3VSQQUogR884dqPwcCWXjPjzBB832hmIkCAgfQ" + password = "0000000000" + } + + certificate_policy { + issuer_parameters { + name = "Self" + } + key_properties { + key_size = 2048 + exportable = true + key_type = "RSA" + reuse_key = false + } + secret_properties { + content_type = "application/x-pkcs12" + } + } + + # The certificate will get replaced with a real one, so we don't want Terrafomr to try and revert it. + lifecycle { + ignore_changes = all + } +} + +data "azurerm_key_vault" "key_vault"{ + name = "kv-${var.tre_id}" + resource_group_name = local.core_resource_group_name +} diff --git a/templates/shared_services/nexus-cert/terraform/locals.tf b/templates/shared_services/nexus-cert/terraform/locals.tf new file mode 100644 index 0000000000..cc1d53f051 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/locals.tf @@ -0,0 +1,25 @@ +locals { + staticweb_storage_name = lower(replace("stwebnexus${var.tre_id}", "-", "")) + + core_resource_group_name = "rg-${var.tre_id}" + + staticweb_backend_pool_name = "beap-nexuscret-staticweb" + app_path_map_name = "upm-nexuscert" + redirect_path_map_name = "upm-nexuscert-redirect" + + insecure_frontend_port_name = "feport-nexuscert-insecure" + secure_frontend_port_name = "feport-nexuscert-secure" + + frontend_ip_configuration_name = "feip-nexuscert-public" + + staticweb_http_setting_name = "be-htst-nexuscert-staticweb" + + insecure_listener_name = "httplstn-nexuscert-insecure" + secure_listener_name = "httplstn-nexuscert-secure" + + redirect_request_routing_rule_name = "rqrt-nexuscert-redirect" + request_routing_rule_name = "rqrt-nexuscert-application" + redirect_configuration_name = "rdrcfg-nexuscert-tosecure" + + certificate_name = "cert-nexus-primary" +} diff --git a/templates/shared_services/nexus-cert/terraform/main.tf b/templates/shared_services/nexus-cert/terraform/main.tf new file mode 100644 index 0000000000..715a8db036 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/main.tf @@ -0,0 +1,15 @@ +# Azure Provider source and version being used +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "=2.97.0" + } + } + + backend "azurerm" {} +} + +provider "azurerm" { + features {} +} diff --git a/templates/shared_services/nexus-cert/terraform/output.tf b/templates/shared_services/nexus-cert/terraform/output.tf new file mode 100644 index 0000000000..17230a0362 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/output.tf @@ -0,0 +1,11 @@ +output "app_gateway_fqdn" { + value = data.azurerm_public_ip.appgwpip_data.fqdn +} + +output "app_gateway_name" { + value = azurerm_application_gateway.agw.name +} + +output "static_web_storage" { + value = azurerm_storage_account.staticweb.name +} \ No newline at end of file diff --git a/templates/shared_services/nexus-cert/terraform/staticweb.tf b/templates/shared_services/nexus-cert/terraform/staticweb.tf new file mode 100644 index 0000000000..fdd4aab6f6 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/staticweb.tf @@ -0,0 +1,30 @@ +data "azurerm_client_config" "deployer" {} + +resource "azurerm_storage_account" "staticweb" { + name = local.staticweb_storage_name + resource_group_name = local.core_resource_group_name + location = var.location + account_kind = "StorageV2" + account_tier = "Standard" + account_replication_type = "LRS" + enable_https_traffic_only = true + allow_blob_public_access = true + + tags = { + tre_id = var.tre_id + } + + static_website { + index_document = "index.html" + error_404_document = "404.html" + } + + lifecycle { ignore_changes = [tags] } +} + +# Assign the "Storage Blob Data Contributor" role needed for uploading certificates to the storage account +resource "azurerm_role_assignment" "stgwriter" { + scope = azurerm_storage_account.staticweb.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.deployer.object_id +} diff --git a/templates/shared_services/nexus-cert/terraform/variables.tf b/templates/shared_services/nexus-cert/terraform/variables.tf new file mode 100644 index 0000000000..c60c7384d2 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/variables.tf @@ -0,0 +1,3 @@ + +variable "tre_id" {} +variable "location" {} From a65b51ae548acbdc5b0aa1c46bd89ce9478b8a9e Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 11 Mar 2022 15:59:07 +0000 Subject: [PATCH 002/142] Replaced webapp with vm --- .../sonatype-nexus/nexus.properties | 1 - .../terraform/cloud-config.yaml | 42 ++++ .../sonatype-nexus/terraform/data.tf | 20 +- .../sonatype-nexus/terraform/vm.tf | 76 +++++++ .../sonatype-nexus/terraform/webapp.tf | 213 ------------------ 5 files changed, 120 insertions(+), 232 deletions(-) delete mode 100644 templates/shared_services/sonatype-nexus/nexus.properties create mode 100644 templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml create mode 100644 templates/shared_services/sonatype-nexus/terraform/vm.tf delete mode 100644 templates/shared_services/sonatype-nexus/terraform/webapp.tf diff --git a/templates/shared_services/sonatype-nexus/nexus.properties b/templates/shared_services/sonatype-nexus/nexus.properties deleted file mode 100644 index 2ae16dd047..0000000000 --- a/templates/shared_services/sonatype-nexus/nexus.properties +++ /dev/null @@ -1 +0,0 @@ -nexus.skipDefaultRepositories=true diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml new file mode 100644 index 0000000000..a5c5f6ec3e --- /dev/null +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -0,0 +1,42 @@ +--- +#cloud-config +package_upgrade: true +apt: + sources: + docker.list: + source: deb [arch=amd64] + https://download.docker.com/linux/ubuntu $RELEASE stable + keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + keyserver: hkp://keyserver.ubuntu.com:80 + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose + - gnupg2 + - pass + +write_files: + # a weekly cron job to have docker free disk space + - path: /etc/cron.weekly/docker-prune + content: | + #!/bin/bash + set -o errexit + docker system prune -f + permissions: '0755' + # ensure Nexus doesn't create default repositories + - path: /etc/nexus-data/nexus.properties + content: | + nexus.skipDefaultRepositories=true + permissions: '0755' + +runcmd: + - export DEBIAN_FRONTEND=noninteractive + - docker pull sonatype/nexus + - docker run -d -v /var/run/docker.sock:/var/run/docker.sock + --restart always --env-file .env + --name nexus + --log-driver local + -p 8081:8081 + sonatype/nexus:oss diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus/terraform/data.tf index 6a0947216d..d7d6244874 100644 --- a/templates/shared_services/sonatype-nexus/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus/terraform/data.tf @@ -3,16 +3,6 @@ data "azurerm_log_analytics_workspace" "tre" { resource_group_name = local.core_resource_group_name } -data "azurerm_app_service_plan" "core" { - name = "plan-${var.tre_id}" - resource_group_name = local.core_resource_group_name -} - -data "azurerm_application_insights" "core" { - name = "appi-${var.tre_id}" - resource_group_name = local.core_resource_group_name -} - data "azurerm_virtual_network" "core" { name = local.core_vnet resource_group_name = local.core_resource_group_name @@ -29,18 +19,12 @@ data "azurerm_subnet" "shared" { name = "SharedSubnet" } -data "azurerm_subnet" "web_app" { - name = "WebAppSubnet" - virtual_network_name = "vnet-${var.tre_id}" - resource_group_name = local.core_resource_group_name -} - data "azurerm_firewall" "fw" { name = "fw-${var.tre_id}" resource_group_name = local.core_resource_group_name } -data "azurerm_private_dns_zone" "azurewebsites" { - name = "privatelink.azurewebsites.net" +data "azurerm_key_vault" "kv" { + name = "kv-${var.tre_id}" resource_group_name = local.core_resource_group_name } diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf new file mode 100644 index 0000000000..2ec38258b5 --- /dev/null +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -0,0 +1,76 @@ +resource "azurerm_network_interface" "internal" { + name = "internal-nic-nexus-${var.tre_id}" + location = var.location + resource_group_name = local.core_resource_group_name + + ip_configuration { + name = "primary" + subnet_id = data.azurerm_subnet.shared.id + private_ip_address_allocation = "Dynamic" + } +} + +resource "random_password" "password" { + length = 16 + lower = true + min_lower = 1 + upper = true + min_upper = 1 + number = true + min_numeric = 1 + special = true + min_special = 1 + override_special = "_%@" +} + +resource "azurerm_key_vault_secret" "nexus_vm_password" { + name = "nexus-vm-password" + value = random_password.password.result + key_vault_id = data.azurerm_key_vault.kv.id +} + +resource "azurerm_linux_virtual_machine" "nexus" { + name = "nexus-${var.tre_id}" + resource_group_name = local.core_resource_group_name + location = var.location + network_interface_ids = [azurerm_network_interface.internal.id] + size = "Standard_B2s" + disable_password_authentication = false + admin_username = "adminuser" + admin_password = random_password.password.result + + custom_data = data.template_cloudinit_config.nexus_config.rendered + + lifecycle { ignore_changes = [tags] } + + source_image_reference { + publisher = "Canonical" + offer = "UbuntuServer" + sku = "18.04-LTS" + version = "latest" + } + + os_disk { + name = "osdisk-nexus-${var.tre_id}" + caching = "ReadWrite" + storage_account_type = "Standard_LRS" + } + + identity { + type = "SystemAssigned" + } +} + +data "template_cloudinit_config" "nexus_config" { + gzip = true + base64_encode = true + + part { + content_type = "text/cloud-config" + content = data.template_file.nexus_config.rendered + } +} + +data "template_file" "nexus_config" { + template = file("${path.module}/cloud-config.yaml") +} diff --git a/templates/shared_services/sonatype-nexus/terraform/webapp.tf b/templates/shared_services/sonatype-nexus/terraform/webapp.tf deleted file mode 100644 index d992c71c61..0000000000 --- a/templates/shared_services/sonatype-nexus/terraform/webapp.tf +++ /dev/null @@ -1,213 +0,0 @@ -resource "azurerm_app_service" "nexus" { - name = "nexus-${var.tre_id}" - resource_group_name = local.core_resource_group_name - location = var.location - app_service_plan_id = data.azurerm_app_service_plan.core.id - https_only = true - - app_settings = { - APPINSIGHTS_INSTRUMENTATIONKEY = data.azurerm_application_insights.core.instrumentation_key - WEBSITES_PORT = "8081" # nexus web-ui listens here - WEBSITES_CONTAINER_START_TIME_LIMIT = "900" # nexus takes a while to start-up - WEBSITE_VNET_ROUTE_ALL = 1 - WEBSITE_DNS_SERVER = "168.63.129.16" # required to access storage over private endpoints - WEBSITES_ENABLE_APP_SERVICE_STORAGE = false - DOCKER_REGISTRY_SERVER_URL = "https://index.docker.io/v1" - } - - lifecycle { ignore_changes = [tags] } - - site_config { - linux_fx_version = "DOCKER|sonatype/nexus3" - remote_debugging_enabled = false - scm_use_main_ip_restriction = true - - always_on = true - min_tls_version = "1.2" - - ip_restriction { - action = "Deny" - ip_address = "0.0.0.0/0" - name = "Deny all" - priority = 2147483647 - } - - websockets_enabled = false - } - - storage_account { - name = "nexus-data" - type = "AzureFiles" - account_name = data.azurerm_storage_account.nexus.name - - access_key = data.azurerm_storage_account.nexus.primary_access_key - share_name = azurerm_storage_share.nexus.name - mount_path = "/nexus-data" - } - - logs { - application_logs { - file_system_level = "Information" - } - - http_logs { - file_system { - retention_in_days = 7 - retention_in_mb = 100 - } - } - } - - # App needs to wait for the properties file to be there - depends_on = [ - null_resource.upload_nexus_props - ] -} - -resource "azurerm_private_endpoint" "nexus_private_endpoint" { - name = "pe-nexus-${var.tre_id}" - resource_group_name = local.core_resource_group_name - location = var.location - subnet_id = data.azurerm_subnet.shared.id - - private_service_connection { - private_connection_resource_id = azurerm_app_service.nexus.id - name = "psc-nexus-${var.tre_id}" - subresource_names = ["sites"] - is_manual_connection = false - } - - private_dns_zone_group { - name = "privatelink.azurewebsites.net" - private_dns_zone_ids = [data.azurerm_private_dns_zone.azurewebsites.id] - } - - lifecycle { ignore_changes = [tags] } -} - -resource "azurerm_app_service_virtual_network_swift_connection" "nexus-integrated-vnet" { - app_service_id = azurerm_app_service.nexus.id - subnet_id = data.azurerm_subnet.web_app.id -} - -resource "azurerm_monitor_diagnostic_setting" "nexus" { - name = "diag-${var.tre_id}" - target_resource_id = azurerm_app_service.nexus.id - log_analytics_workspace_id = data.azurerm_log_analytics_workspace.tre.id - - log { - category = "AppServiceHTTPLogs" - enabled = true - - retention_policy { - days = 1 - enabled = false - } - } - - log { - category = "AppServiceConsoleLogs" - enabled = true - - retention_policy { - days = 1 - enabled = false - } - } - - log { - category = "AppServiceAppLogs" - enabled = true - - retention_policy { - days = 1 - enabled = false - } - } - - log { - category = "AppServiceFileAuditLogs" - enabled = true - - retention_policy { - days = 1 - enabled = false - } - } - - log { - category = "AppServiceAuditLogs" - enabled = true - - retention_policy { - days = 1 - enabled = false - } - } - - log { - category = "AppServiceIPSecAuditLogs" - enabled = true - - retention_policy { - days = 1 - enabled = false - } - } - - log { - category = "AppServicePlatformLogs" - enabled = true - - retention_policy { - days = 1 - enabled = false - } - } - - log { - category = "AppServiceAntivirusScanAuditLogs" - enabled = true - - retention_policy { - days = 1 - enabled = false - } - } - - metric { - category = "AllMetrics" - enabled = true - - retention_policy { - enabled = false - } - } -} - -resource "azurerm_storage_share" "nexus" { - name = "nexus-data" - storage_account_name = data.azurerm_storage_account.nexus.name - quota = var.nexus_storage_limit -} - -# Include a properties file in the nexus-data path will change its configuration. We need this to instruct it not to create default repositories. -resource "null_resource" "upload_nexus_props" { - provisioner "local-exec" { - command = < Date: Fri, 11 Mar 2022 16:08:39 +0000 Subject: [PATCH 003/142] Amended docker start commands --- .../shared_services/sonatype-nexus/terraform/cloud-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index a5c5f6ec3e..5dcd07b494 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -34,8 +34,9 @@ write_files: runcmd: - export DEBIAN_FRONTEND=noninteractive - docker pull sonatype/nexus + - docker build –rm –tag sonatype/nexus oss/ - docker run -d -v /var/run/docker.sock:/var/run/docker.sock - --restart always --env-file .env + --restart always --name nexus --log-driver local -p 8081:8081 From cdc66a2683f70b01b8e96a3ca0775e4a12465ec7 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 11 Mar 2022 16:14:43 +0000 Subject: [PATCH 004/142] Amended firewall --- .../shared_services/sonatype-nexus/terraform/firewall.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/firewall.tf b/templates/shared_services/sonatype-nexus/terraform/firewall.tf index 17113d48db..7649726d82 100644 --- a/templates/shared_services/sonatype-nexus/terraform/firewall.tf +++ b/templates/shared_services/sonatype-nexus/terraform/firewall.tf @@ -1,5 +1,5 @@ -resource "azurerm_firewall_application_rule_collection" "web_app_subnet_nexus" { - name = "arc-web_app_subnet_nexus" +resource "azurerm_firewall_application_rule_collection" "vm_subnet_nexus" { + name = "vm_subnet_nexus" azure_firewall_name = data.azurerm_firewall.fw.name resource_group_name = data.azurerm_firewall.fw.resource_group_name priority = 104 @@ -17,6 +17,6 @@ resource "azurerm_firewall_application_rule_collection" "web_app_subnet_nexus" { } target_fqdns = local.nexus_allowed_fqdns_list - source_addresses = data.azurerm_subnet.web_app.address_prefixes + source_addresses = data.azurerm_subnet.shared.address_prefixes } } From 87761b74c4e7828f00d72f4c60cc9d910fc5d496 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 11 Mar 2022 16:46:20 +0000 Subject: [PATCH 005/142] Add nexus config to persistent volume --- .../sonatype-nexus/terraform/cloud-config.yaml | 10 +++++----- .../sonatype-nexus/terraform/locals.tf | 1 - .../sonatype-nexus/terraform/variables.tf | 12 ------------ 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 5dcd07b494..6a0c2a98ec 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -33,11 +33,11 @@ write_files: runcmd: - export DEBIAN_FRONTEND=noninteractive - - docker pull sonatype/nexus - - docker build –rm –tag sonatype/nexus oss/ - - docker run -d -v /var/run/docker.sock:/var/run/docker.sock + # Run the nexus container (exposing port 8081 and mapping volume for nexus config) + - docker run -d + -p 8081:8081 + -v /etc/nexus-data:/nexus-data --restart always --name nexus --log-driver local - -p 8081:8081 - sonatype/nexus:oss + sonatype/nexus3 diff --git a/templates/shared_services/sonatype-nexus/terraform/locals.tf b/templates/shared_services/sonatype-nexus/terraform/locals.tf index 015df1fe39..50a394f2b9 100644 --- a/templates/shared_services/sonatype-nexus/terraform/locals.tf +++ b/templates/shared_services/sonatype-nexus/terraform/locals.tf @@ -2,6 +2,5 @@ locals { core_vnet = "vnet-${var.tre_id}" core_resource_group_name = "rg-${var.tre_id}" firewall_name = "fw-${var.tre_id}" - storage_account_name = lower(replace("stg-${var.tre_id}", "-", "")) nexus_allowed_fqdns_list = distinct(compact(split(",", replace(var.nexus_allowed_fqdns, " ", "")))) } diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus/terraform/variables.tf index 3f2d9ca752..ec8473681b 100644 --- a/templates/shared_services/sonatype-nexus/terraform/variables.tf +++ b/templates/shared_services/sonatype-nexus/terraform/variables.tf @@ -8,20 +8,8 @@ variable "location" { description = "Azure location (region) for deployment of core TRE services" } -variable "nexus_storage_limit" { - type = number - description = "Space allocated in GB for the Nexus data in Azure Files Share" - default = 1024 -} - variable "nexus_allowed_fqdns" { type = string description = "comma seperated string of allowed FQDNs for Nexus" default = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com,repo.anaconda.com" } - -variable "nexus_properties_path" { - type = string - description = "relative path of nexus properties file" - default = "/cnab/app/nexus.properties" -} From d14ec955b44bfdaa54e735b200618e44288728ca Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 11 Mar 2022 21:17:05 +0000 Subject: [PATCH 006/142] Add private dns zone --- templates/core/terraform/.terraform.lock.hcl | 27 +++++++------- .../gitea/terraform/.terraform.lock.hcl | 27 +++++++------- .../terraform/.terraform.lock.hcl | 35 +++++++++++++++++++ .../sonatype-nexus/terraform/data.tf | 5 --- .../sonatype-nexus/terraform/output.tf | 4 +-- .../sonatype-nexus/terraform/vm.tf | 24 ++++++++++++- 6 files changed, 88 insertions(+), 34 deletions(-) diff --git a/templates/core/terraform/.terraform.lock.hcl b/templates/core/terraform/.terraform.lock.hcl index 0595b21878..2b4848bd6d 100644 --- a/templates/core/terraform/.terraform.lock.hcl +++ b/templates/core/terraform/.terraform.lock.hcl @@ -21,20 +21,21 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.1.0" + version = "2.2.2" hashes = [ - "h1:EYZdckuGU3n6APs97nS2LxZm3dDtGqyM4qaIvsmac8o=", - "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", - "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", - "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", - "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", - "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", - "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", - "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", - "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", - "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", - "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", - "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", + "h1:5UYW2wJ320IggrzLt8tLD6MowePqycWtH1b2RInHZkE=", + "zh:027e4873c69da214e2fed131666d5de92089732a11d096b68257da54d30b6f9d", + "zh:0ba2216e16cfb72538d76a4c4945b4567a76f7edbfef926b1c5a08d7bba2a043", + "zh:1fee8f6aae1833c27caa96e156cf99a681b6f085e476d7e1b77d285e21d182c1", + "zh:2e8a3e72e877003df1c390a231e0d8e827eba9f788606e643f8e061218750360", + "zh:719008f9e262aa1523a6f9132adbe9eee93c648c2981f8359ce41a40e6425433", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9a70fdbe6ef955c4919a4519caca116f34c19c7ddedd77990fbe4f80fe66dc84", + "zh:abc412423d670cbb6264827fa80e1ffdc4a74aff3f19ba6a239dd87b85b15bec", + "zh:ae953a62c94d2a2a0822e5717fafc54e454af57bd6ed02cd301b9786765c1dd3", + "zh:be0910bdf46698560f9e86f51a4ff795c62c02f8dc82b2b1dab77a0b3a93f61e", + "zh:e58f9083b7971919b95f553227adaa7abe864fce976f0166cf4d65fc17257ff2", + "zh:ff4f77cbdbb22cc98182821c7ef84dce16298ab0e997d5c7fae97247f7a4bcb0", ] } diff --git a/templates/shared_services/gitea/terraform/.terraform.lock.hcl b/templates/shared_services/gitea/terraform/.terraform.lock.hcl index 00b91198f9..0535930973 100644 --- a/templates/shared_services/gitea/terraform/.terraform.lock.hcl +++ b/templates/shared_services/gitea/terraform/.terraform.lock.hcl @@ -21,20 +21,21 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.1.0" + version = "2.2.2" hashes = [ - "h1:EYZdckuGU3n6APs97nS2LxZm3dDtGqyM4qaIvsmac8o=", - "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", - "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", - "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", - "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", - "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", - "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", - "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", - "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", - "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", - "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", - "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", + "h1:5UYW2wJ320IggrzLt8tLD6MowePqycWtH1b2RInHZkE=", + "zh:027e4873c69da214e2fed131666d5de92089732a11d096b68257da54d30b6f9d", + "zh:0ba2216e16cfb72538d76a4c4945b4567a76f7edbfef926b1c5a08d7bba2a043", + "zh:1fee8f6aae1833c27caa96e156cf99a681b6f085e476d7e1b77d285e21d182c1", + "zh:2e8a3e72e877003df1c390a231e0d8e827eba9f788606e643f8e061218750360", + "zh:719008f9e262aa1523a6f9132adbe9eee93c648c2981f8359ce41a40e6425433", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9a70fdbe6ef955c4919a4519caca116f34c19c7ddedd77990fbe4f80fe66dc84", + "zh:abc412423d670cbb6264827fa80e1ffdc4a74aff3f19ba6a239dd87b85b15bec", + "zh:ae953a62c94d2a2a0822e5717fafc54e454af57bd6ed02cd301b9786765c1dd3", + "zh:be0910bdf46698560f9e86f51a4ff795c62c02f8dc82b2b1dab77a0b3a93f61e", + "zh:e58f9083b7971919b95f553227adaa7abe864fce976f0166cf4d65fc17257ff2", + "zh:ff4f77cbdbb22cc98182821c7ef84dce16298ab0e997d5c7fae97247f7a4bcb0", ] } diff --git a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl b/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl index 6ce0e29592..2fe9d1fc15 100644 --- a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl +++ b/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl @@ -37,3 +37,38 @@ provider "registry.terraform.io/hashicorp/null" { "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", ] } + +provider "registry.terraform.io/hashicorp/random" { + version = "3.1.0" + hashes = [ + "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", + "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", + "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", + "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", + "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", + "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", + "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", + "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", + "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", + "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", + "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", + "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus/terraform/data.tf index d7d6244874..9bd6230220 100644 --- a/templates/shared_services/sonatype-nexus/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus/terraform/data.tf @@ -8,11 +8,6 @@ data "azurerm_virtual_network" "core" { resource_group_name = local.core_resource_group_name } -data "azurerm_storage_account" "nexus" { - name = local.storage_account_name - resource_group_name = local.core_resource_group_name -} - data "azurerm_subnet" "shared" { resource_group_name = local.core_resource_group_name virtual_network_name = local.core_vnet diff --git a/templates/shared_services/sonatype-nexus/terraform/output.tf b/templates/shared_services/sonatype-nexus/terraform/output.tf index d69d8570a5..b92393b369 100644 --- a/templates/shared_services/sonatype-nexus/terraform/output.tf +++ b/templates/shared_services/sonatype-nexus/terraform/output.tf @@ -1,3 +1,3 @@ output "nexus_fqdn" { - value = azurerm_app_service.nexus.default_site_hostname -} \ No newline at end of file + value = azurerm_private_dns_a_record.nexus_vm.fqdn +} diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 2ec38258b5..2f616ea5a7 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -6,10 +6,32 @@ resource "azurerm_network_interface" "internal" { ip_configuration { name = "primary" subnet_id = data.azurerm_subnet.shared.id - private_ip_address_allocation = "Dynamic" + private_ip_address_allocation = "Static" } } +resource "azurerm_private_dns_zone" "cloudapp" { + name = "${var.location}.cloudapp.azure.com" + resource_group_name = var.resource_group_name + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_private_dns_zone_virtual_network_link" "cloudapp" { + name = "cloudapp" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.cloudapp.name + virtual_network_id = data.azurerm_virtual_network.core.id +} + +resource "azurerm_private_dns_a_record" "nexus_vm" { + name = "nexus-${tre_id}" + zone_name = azurerm_private_dns_zone.cloudapp.name + resource_group_name = var.resource_group_name + ttl = 300 + records = [azurerm_linux_virtual_machine.nexus.private_ip_address] +} + resource "random_password" "password" { length = 16 lower = true From a0490693bf2a31640a7644f24b54d658cf220099 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 14 Mar 2022 10:39:00 +0000 Subject: [PATCH 007/142] Corrected rg var --- .../shared_services/sonatype-nexus/terraform/vm.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 2f616ea5a7..3bc2010056 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -6,28 +6,28 @@ resource "azurerm_network_interface" "internal" { ip_configuration { name = "primary" subnet_id = data.azurerm_subnet.shared.id - private_ip_address_allocation = "Static" + private_ip_address_allocation = "Dynamic" } } resource "azurerm_private_dns_zone" "cloudapp" { name = "${var.location}.cloudapp.azure.com" - resource_group_name = var.resource_group_name + resource_group_name = local.core_resource_group_name lifecycle { ignore_changes = [tags] } } resource "azurerm_private_dns_zone_virtual_network_link" "cloudapp" { name = "cloudapp" - resource_group_name = var.resource_group_name + resource_group_name = local.core_resource_group_name private_dns_zone_name = azurerm_private_dns_zone.cloudapp.name virtual_network_id = data.azurerm_virtual_network.core.id } resource "azurerm_private_dns_a_record" "nexus_vm" { - name = "nexus-${tre_id}" + name = "nexus-${var.tre_id}" zone_name = azurerm_private_dns_zone.cloudapp.name - resource_group_name = var.resource_group_name + resource_group_name = local.core_resource_group_name ttl = 300 records = [azurerm_linux_virtual_machine.nexus.private_ip_address] } From cb5b2285590b39ac7ab40fc444ead8b3a19d9cbe Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Tue, 15 Mar 2022 10:01:19 +0000 Subject: [PATCH 008/142] Added Nexus letsencrypt cert gen --- Makefile | 12 ++++- templates/core/terraform/.terraform.lock.hcl | 27 +++++------ .../core/terraform/scripts/letsencrypt.sh | 2 + .../nexus-cert/scripts/auth-hook.sh | 16 +++++++ .../nexus-cert/scripts/cleanup-hook.sh | 1 + .../nexus-cert/scripts/json-to-env.sh | 46 +++++++++++++++++++ .../nexus-cert/scripts/letsencrypt.sh | 10 ++-- .../nexus-cert/scripts/outputs.sh | 18 ++++++++ .../nexus-cert/terraform/output.tf | 16 +++++-- 9 files changed, 124 insertions(+), 24 deletions(-) create mode 100644 templates/shared_services/nexus-cert/scripts/auth-hook.sh create mode 100644 templates/shared_services/nexus-cert/scripts/cleanup-hook.sh create mode 100644 templates/shared_services/nexus-cert/scripts/json-to-env.sh create mode 100644 templates/shared_services/nexus-cert/scripts/outputs.sh diff --git a/Makefile b/Makefile index 0629911037..5c22f166ad 100644 --- a/Makefile +++ b/Makefile @@ -136,12 +136,22 @@ gitea-install: nexus-install: $(call target_title, "Installing Nexus") \ && make SHARED_SERVICE_KEY=shared-service-sonatype-nexus TF_VAR_nexus_properties_path=../nexus.properties terraform-shared-service-deploy DIR=./templates/shared_services/sonatype-nexus/terraform +# / End migration targets nexus-cert-install: $(call target_title, "Installing Nexus Cert") \ && make SHARED_SERVICE_KEY=shared-service-nexus-cert terraform-shared-service-deploy DIR=./templates/shared_services/nexus-cert/terraform -# / End migration targets +nexus-letsencrypt: + $(call target_title, "Requesting LetsEncrypt SSL certificate for Nexus") \ + && . ./devops/scripts/check_dependencies.sh nodocker,certbot \ + && . ./devops/scripts/load_env.sh ./templates/core/.env \ + && . ./devops/scripts/load_env.sh ./devops/.env \ + && . ./devops/scripts/load_terraform_env.sh ./devops/.env \ + && . ./devops/scripts/load_terraform_env.sh ./templates/core/.env \ + && pushd ./templates/shared_services/nexus-cert/scripts/ > /dev/null && . ./outputs.sh && popd > /dev/null \ + && . ./devops/scripts/load_env.sh ./templates/shared_services/nexus-cert/private.env \ + && ./templates/shared_services/nexus-cert/scripts/letsencrypt.sh deploy-core: tre-start $(call target_title, "Deploying TRE") \ diff --git a/templates/core/terraform/.terraform.lock.hcl b/templates/core/terraform/.terraform.lock.hcl index 0595b21878..2b4848bd6d 100644 --- a/templates/core/terraform/.terraform.lock.hcl +++ b/templates/core/terraform/.terraform.lock.hcl @@ -21,20 +21,21 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.1.0" + version = "2.2.2" hashes = [ - "h1:EYZdckuGU3n6APs97nS2LxZm3dDtGqyM4qaIvsmac8o=", - "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", - "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", - "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", - "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", - "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", - "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", - "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", - "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", - "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", - "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", - "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", + "h1:5UYW2wJ320IggrzLt8tLD6MowePqycWtH1b2RInHZkE=", + "zh:027e4873c69da214e2fed131666d5de92089732a11d096b68257da54d30b6f9d", + "zh:0ba2216e16cfb72538d76a4c4945b4567a76f7edbfef926b1c5a08d7bba2a043", + "zh:1fee8f6aae1833c27caa96e156cf99a681b6f085e476d7e1b77d285e21d182c1", + "zh:2e8a3e72e877003df1c390a231e0d8e827eba9f788606e643f8e061218750360", + "zh:719008f9e262aa1523a6f9132adbe9eee93c648c2981f8359ce41a40e6425433", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9a70fdbe6ef955c4919a4519caca116f34c19c7ddedd77990fbe4f80fe66dc84", + "zh:abc412423d670cbb6264827fa80e1ffdc4a74aff3f19ba6a239dd87b85b15bec", + "zh:ae953a62c94d2a2a0822e5717fafc54e454af57bd6ed02cd301b9786765c1dd3", + "zh:be0910bdf46698560f9e86f51a4ff795c62c02f8dc82b2b1dab77a0b3a93f61e", + "zh:e58f9083b7971919b95f553227adaa7abe864fce976f0166cf4d65fc17257ff2", + "zh:ff4f77cbdbb22cc98182821c7ef84dce16298ab0e997d5c7fae97247f7a4bcb0", ] } diff --git a/templates/core/terraform/scripts/letsencrypt.sh b/templates/core/terraform/scripts/letsencrypt.sh index 0bd4734a16..68892db8b1 100755 --- a/templates/core/terraform/scripts/letsencrypt.sh +++ b/templates/core/terraform/scripts/letsencrypt.sh @@ -3,6 +3,8 @@ set -e script_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +echo $FQDN +exit if [[ -z ${STORAGE_ACCOUNT} ]]; then echo "STORAGE_ACCOUNT not set" exit 1 diff --git a/templates/shared_services/nexus-cert/scripts/auth-hook.sh b/templates/shared_services/nexus-cert/scripts/auth-hook.sh new file mode 100644 index 0000000000..ede440f231 --- /dev/null +++ b/templates/shared_services/nexus-cert/scripts/auth-hook.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +cat << EOF > 'validation.txt' +${CERTBOT_VALIDATION} +EOF + +az storage blob upload \ + --account-name "${STORAGE_ACCOUNT}" \ + --auth-mode login \ + --container-name '$web' \ + --file 'validation.txt' \ + --name ".well-known/acme-challenge/${CERTBOT_TOKEN}" \ + --no-progress \ + --only-show-errors + +sleep 10s diff --git a/templates/shared_services/nexus-cert/scripts/cleanup-hook.sh b/templates/shared_services/nexus-cert/scripts/cleanup-hook.sh new file mode 100644 index 0000000000..a9bf588e2f --- /dev/null +++ b/templates/shared_services/nexus-cert/scripts/cleanup-hook.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/templates/shared_services/nexus-cert/scripts/json-to-env.sh b/templates/shared_services/nexus-cert/scripts/json-to-env.sh new file mode 100644 index 0000000000..923c4fc193 --- /dev/null +++ b/templates/shared_services/nexus-cert/scripts/json-to-env.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -e + +echo "# Generated environment variables from tf output" + +jq -r ' + [ + { + "path": "fqdn", + "env_var": "FQDN" + }, + { + "path": "application_gateway", + "env_var": "APPLICATION_GATEWAY" + }, + { + "path": "storage_account", + "env_var": "STORAGE_ACCOUNT" + }, + { + "path": "resource_group_name", + "env_var": "RESOURCE_GROUP_NAME" + }, + { + "path": "keyvault", + "env_var": "KEYVAULT" + } + ] + as $env_vars_to_extract + | + with_entries( + select ( + .key as $a + | + any( $env_vars_to_extract[]; .path == $a) + ) + | + .key |= . as $old_key | ($env_vars_to_extract[] | select (.path == $old_key) | .env_var) + ) + | + to_entries + | + map("\(.key)=\"\(.value.value)\"") + | + .[] + ' | sed "s/\"/'/g" # replace double quote with single quote to handle special chars diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh index d2bcddd26b..2c36deb523 100644 --- a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh +++ b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh @@ -2,13 +2,11 @@ set -e script_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")) - if [[ -z ${STORAGE_ACCOUNT} ]]; then echo "STORAGE_ACCOUNT not set" exit 1 fi - echo "Checking for index.html file in storage account" # Create the default index.html page @@ -43,8 +41,7 @@ else echo "index.html already present" fi -ledir=$(pwd)/letsencrypt - +ledir=$(cd ${script_dir} && cd ../ && pwd)/letsencrypt mkdir -p "${ledir}/logs" # Initiate the ACME challange @@ -56,13 +53,14 @@ mkdir -p "${ledir}/logs" --preferred-challenges=http \ --manual-auth-hook ${script_dir}/auth-hook.sh \ --manual-cleanup-hook ${script_dir}/cleanup-hook.sh \ - --domain $FQDN \ + --domain ${FQDN} \ --non-interactive \ --agree-tos \ --register-unsafely-without-email + # Convert the generated certificate to a .pfx -CERT_DIR="${ledir}/live/$FQDN" +CERT_DIR="${ledir}/live/${FQDN}" CERT_PASSWORD=$(openssl rand -base64 30) openssl pkcs12 -export \ -inkey "${CERT_DIR}/privkey.pem" \ diff --git a/templates/shared_services/nexus-cert/scripts/outputs.sh b/templates/shared_services/nexus-cert/scripts/outputs.sh new file mode 100644 index 0000000000..5a94968ba1 --- /dev/null +++ b/templates/shared_services/nexus-cert/scripts/outputs.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +if [ ! -f ../tre_output.json ]; then + # Connect to the remote backend of Terraform + export TF_LOG="" + terraform init -input=false -backend=true -reconfigure -upgrade \ + -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ + -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ + -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ + -backend-config="key=${TRE_ID}" + + # Convert the output to json + terraform output -json > ../tre_output.json +fi + +# Now create an .env file +./json-to-env.sh < ../tre_output.json > ../private.env diff --git a/templates/shared_services/nexus-cert/terraform/output.tf b/templates/shared_services/nexus-cert/terraform/output.tf index 17230a0362..30faf2a47a 100644 --- a/templates/shared_services/nexus-cert/terraform/output.tf +++ b/templates/shared_services/nexus-cert/terraform/output.tf @@ -1,11 +1,19 @@ -output "app_gateway_fqdn" { +output "fqdn" { value = data.azurerm_public_ip.appgwpip_data.fqdn } -output "app_gateway_name" { +output "application_gateway" { value = azurerm_application_gateway.agw.name } -output "static_web_storage" { +output "storage_account" { value = azurerm_storage_account.staticweb.name -} \ No newline at end of file +} + +output "resource_group_name" { + value = azurerm_application_gateway.agw.resource_group_name +} + +output "keyvault" { + value = data.azurerm_key_vault.key_vault.name +} From 2f91623eaf990bfdd2f169dfc39ec0aabb74a0f8 Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Tue, 15 Mar 2022 10:32:50 +0000 Subject: [PATCH 009/142] Fixed linting --- templates/shared_services/nexus-cert/terraform/appgateway.tf | 2 +- templates/shared_services/nexus-cert/terraform/certificate.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/shared_services/nexus-cert/terraform/appgateway.tf b/templates/shared_services/nexus-cert/terraform/appgateway.tf index c23c357034..4ca553d406 100644 --- a/templates/shared_services/nexus-cert/terraform/appgateway.tf +++ b/templates/shared_services/nexus-cert/terraform/appgateway.tf @@ -159,7 +159,7 @@ resource "azurerm_application_gateway" "agw" { } data "azurerm_subnet" "app_gw_subnet" { - name = "AppGwSubnet" + name = "AppGwSubnet" virtual_network_name = "vnet-${var.tre_id}" resource_group_name = local.core_resource_group_name } diff --git a/templates/shared_services/nexus-cert/terraform/certificate.tf b/templates/shared_services/nexus-cert/terraform/certificate.tf index fabcd7c972..a7563916e7 100644 --- a/templates/shared_services/nexus-cert/terraform/certificate.tf +++ b/templates/shared_services/nexus-cert/terraform/certificate.tf @@ -43,7 +43,7 @@ resource "azurerm_key_vault_certificate" "tlscert" { } } -data "azurerm_key_vault" "key_vault"{ +data "azurerm_key_vault" "key_vault" { name = "kv-${var.tre_id}" resource_group_name = local.core_resource_group_name } From 073d7808c21ca961382a2e2ade060f7466ddcbe6 Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Tue, 15 Mar 2022 10:44:01 +0000 Subject: [PATCH 010/142] Changed terraform.lock.hcl to previous version --- .../gitea/terraform/.terraform.lock.hcl | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/templates/shared_services/gitea/terraform/.terraform.lock.hcl b/templates/shared_services/gitea/terraform/.terraform.lock.hcl index fe444a7bf4..00b91198f9 100644 --- a/templates/shared_services/gitea/terraform/.terraform.lock.hcl +++ b/templates/shared_services/gitea/terraform/.terraform.lock.hcl @@ -21,21 +21,20 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.2.1" + version = "2.1.0" hashes = [ - "h1:y1SV7/L0B2/q30waki+emxW2+e8+fyEv+m53nwH6ME4=", - "zh:15282174d8e0644a86c21c25d4ea1eaff2950fffc5eeb0281cbebd74c13cfd06", - "zh:46cd90f69cfd7dad613dc71606e25c339cdaabe8d5ebc1ad712c4c30747ec0fd", - "zh:518519a26a709b7a8ca2f9282389d2ac08eab057e492805655970e7eca25cab8", - "zh:5ba89d886ada09aee3926d8340853d0bf88953f3ede2cf6d7af3f2a41fec642e", - "zh:6a20f6ca3a24af94c88a5b9afa50d4fd7d47c39ee8c0184c415aaae14204c497", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a3f65fda390fcbfa7368f8d6da4d261b64383f3c86b5667e07a2176e27278cda", - "zh:a5f3ab57445b1974d3064ed223d3c4b21e89d375e46384fed65c199241003b76", - "zh:a8fae6bd2d1b233c0cf5690d68e29930d73fbf167dd61499597ed4acadf1f4a2", - "zh:b3c848292d0ca01f7da81a5a945ab939817bac032110d9a1b07d8264490db1b1", - "zh:e0fdc5cc40cb9c0726d2a195a1e48f5e43bc33f3b3f6dfbda72ca7010bc00a75", - "zh:ef6139bfa8bcbc2ed6f07237362ce347a90500722314676e1203cf6c8f8ef2ad", + "h1:EYZdckuGU3n6APs97nS2LxZm3dDtGqyM4qaIvsmac8o=", + "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", + "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", + "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", + "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", + "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", + "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", + "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", + "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", + "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", + "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", + "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", ] } From f0679416674ed09bd8783d6ae55ce5e26a3a2837 Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Tue, 15 Mar 2022 15:18:43 +0000 Subject: [PATCH 011/142] Removed leftover debug --- templates/core/terraform/scripts/letsencrypt.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/core/terraform/scripts/letsencrypt.sh b/templates/core/terraform/scripts/letsencrypt.sh index 68892db8b1..0bd4734a16 100755 --- a/templates/core/terraform/scripts/letsencrypt.sh +++ b/templates/core/terraform/scripts/letsencrypt.sh @@ -3,8 +3,6 @@ set -e script_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")) -echo $FQDN -exit if [[ -z ${STORAGE_ACCOUNT} ]]; then echo "STORAGE_ACCOUNT not set" exit 1 From f728fcd7ba6cc0cd10cd913d47f09200515b6062 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Wed, 16 Mar 2022 08:50:36 +0000 Subject: [PATCH 012/142] Typo fix Co-authored-by: Stuart Leeks --- templates/shared_services/nexus-cert/terraform/certificate.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/nexus-cert/terraform/certificate.tf b/templates/shared_services/nexus-cert/terraform/certificate.tf index a7563916e7..71d8afea68 100644 --- a/templates/shared_services/nexus-cert/terraform/certificate.tf +++ b/templates/shared_services/nexus-cert/terraform/certificate.tf @@ -37,7 +37,7 @@ resource "azurerm_key_vault_certificate" "tlscert" { } } - # The certificate will get replaced with a real one, so we don't want Terrafomr to try and revert it. + # The certificate will get replaced with a real one, so we don't want Terraform to try and revert it. lifecycle { ignore_changes = all } From c76d76b35018d998726382cb96521a07f138553a Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Wed, 16 Mar 2022 08:51:23 +0000 Subject: [PATCH 013/142] File path amend Co-authored-by: Stuart Leeks --- templates/shared_services/nexus-cert/scripts/letsencrypt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh index 2c36deb523..eb4e3c4777 100644 --- a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh +++ b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh @@ -41,7 +41,7 @@ else echo "index.html already present" fi -ledir=$(cd ${script_dir} && cd ../ && pwd)/letsencrypt +ledir="${script_dir}/../letsencrypt" mkdir -p "${ledir}/logs" # Initiate the ACME challange From b4ea8dc4f9aff2a253143fe9fa0157e8f127fbcc Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 16 Mar 2022 15:26:15 +0000 Subject: [PATCH 014/142] Fix for cloudapp DNS resolution errors --- .../terraform/.terraform.lock.hcl | 18 ------------------ .../sonatype-nexus/terraform/vm.tf | 18 +++++++++--------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl b/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl index 2fe9d1fc15..61c54c45c8 100644 --- a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl +++ b/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl @@ -20,24 +20,6 @@ provider "registry.terraform.io/hashicorp/azurerm" { ] } -provider "registry.terraform.io/hashicorp/null" { - version = "3.1.0" - hashes = [ - "h1:vpC6bgUQoJ0znqIKVFevOdq+YQw42bRq0u+H3nto8nA=", - "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", - "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", - "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", - "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", - "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", - "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", - "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", - "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", - "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", - "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", - "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", - ] -} - provider "registry.terraform.io/hashicorp/random" { version = "3.1.0" hashes = [ diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 3bc2010056..d471e68bf1 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -1,4 +1,4 @@ -resource "azurerm_network_interface" "internal" { +resource "azurerm_network_interface" "nexus" { name = "internal-nic-nexus-${var.tre_id}" location = var.location resource_group_name = local.core_resource_group_name @@ -10,23 +10,23 @@ resource "azurerm_network_interface" "internal" { } } -resource "azurerm_private_dns_zone" "cloudapp" { - name = "${var.location}.cloudapp.azure.com" +resource "azurerm_private_dns_zone" "nexus" { + name = "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" resource_group_name = local.core_resource_group_name lifecycle { ignore_changes = [tags] } } -resource "azurerm_private_dns_zone_virtual_network_link" "cloudapp" { - name = "cloudapp" +resource "azurerm_private_dns_zone_virtual_network_link" "nexus" { + name = "nexus" resource_group_name = local.core_resource_group_name - private_dns_zone_name = azurerm_private_dns_zone.cloudapp.name + private_dns_zone_name = azurerm_private_dns_zone.nexus.name virtual_network_id = data.azurerm_virtual_network.core.id } resource "azurerm_private_dns_a_record" "nexus_vm" { - name = "nexus-${var.tre_id}" - zone_name = azurerm_private_dns_zone.cloudapp.name + name = "@" + zone_name = azurerm_private_dns_zone.nexus.name resource_group_name = local.core_resource_group_name ttl = 300 records = [azurerm_linux_virtual_machine.nexus.private_ip_address] @@ -55,7 +55,7 @@ resource "azurerm_linux_virtual_machine" "nexus" { name = "nexus-${var.tre_id}" resource_group_name = local.core_resource_group_name location = var.location - network_interface_ids = [azurerm_network_interface.internal.id] + network_interface_ids = [azurerm_network_interface.nexus.id] size = "Standard_B2s" disable_password_authentication = false admin_username = "adminuser" From 72f468e4a81c439488216619f56af4f813b3afa4 Mon Sep 17 00:00:00 2001 From: ross-p-smith Date: Thu, 17 Mar 2022 10:55:55 +0000 Subject: [PATCH 015/142] Docker running on Nexus VM --- templates/core/terraform/.terraform.lock.hcl | 27 ++++++++++--------- .../gitea/terraform/.terraform.lock.hcl | 27 ++++++++++--------- .../terraform/.terraform.lock.hcl | 27 ++++++++++--------- .../terraform/cloud-config.yaml | 10 +++++++ .../sonatype-nexus/terraform/data.tf | 5 ++++ .../sonatype-nexus/terraform/locals.tf | 1 + .../sonatype-nexus/terraform/variables.tf | 2 +- .../sonatype-nexus/terraform/vm.tf | 4 +++ 8 files changed, 63 insertions(+), 40 deletions(-) diff --git a/templates/core/terraform/.terraform.lock.hcl b/templates/core/terraform/.terraform.lock.hcl index 9fc8c04f44..3505fcea5d 100644 --- a/templates/core/terraform/.terraform.lock.hcl +++ b/templates/core/terraform/.terraform.lock.hcl @@ -58,20 +58,21 @@ provider "registry.terraform.io/hashicorp/local" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.1.0" + version = "3.1.1" hashes = [ - "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", - "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", - "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", - "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", - "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", - "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", - "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", - "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", - "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", - "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", - "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", - "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + "h1:wK4ig6D02sQtpYs82HNJ8RnYeMcfz+eQMjesc+b4QeE=", + "zh:04dbe3a562a5da087aca7729332dc4bdba763e71a7a9fcf2b71d38454af8975d", + "zh:23ad79fbd96be6271560cb78855b51129845a1a52edff19f6725f99851e36367", + "zh:34678e44716d76bfad398506757680b3285bf8b2704e0637fcec9c3f3b659f85", + "zh:46c64dfa7c7c48576240cda9a022e84e4a4bd049d22bcb8e5a72b9ceebf24838", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b6a9d785c5d1fbd18d3f9f7130ce8a9f3b95ec3e14ba0f1a5ae557f50b2b58a", + "zh:be00333ae3bbb520d205d8dcf7ddb84fad60cbe0104b6f45bb1fa8d37ad1fbfb", + "zh:cce05ea51cb86ca5cc4bd11a6f9d7c0463cdf74bc45bccbdcbc2b3a6c6465248", + "zh:d5d065af722ad73874f952c0a0d75a35e7c58bceba153b2411e869ad41ff0aa8", + "zh:e8e7abadccca7f484f4e8bb4d2639ba3e89efa94f1d456eb5f173f7053127d63", + "zh:f6b9d9947026ea4e7bb9a2e7c4f15a57dbf26d7b3799df24405a3e7bc0d9ea00", + "zh:ff0cfd9c52c724aef6ffda20285b7fb77b6b044f39308a95dc4f993993802bb4", ] } diff --git a/templates/shared_services/gitea/terraform/.terraform.lock.hcl b/templates/shared_services/gitea/terraform/.terraform.lock.hcl index 0535930973..11ffd4227a 100644 --- a/templates/shared_services/gitea/terraform/.terraform.lock.hcl +++ b/templates/shared_services/gitea/terraform/.terraform.lock.hcl @@ -40,19 +40,20 @@ provider "registry.terraform.io/hashicorp/local" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.1.0" + version = "3.1.1" hashes = [ - "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", - "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", - "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", - "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", - "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", - "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", - "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", - "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", - "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", - "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", - "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", - "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + "h1:wK4ig6D02sQtpYs82HNJ8RnYeMcfz+eQMjesc+b4QeE=", + "zh:04dbe3a562a5da087aca7729332dc4bdba763e71a7a9fcf2b71d38454af8975d", + "zh:23ad79fbd96be6271560cb78855b51129845a1a52edff19f6725f99851e36367", + "zh:34678e44716d76bfad398506757680b3285bf8b2704e0637fcec9c3f3b659f85", + "zh:46c64dfa7c7c48576240cda9a022e84e4a4bd049d22bcb8e5a72b9ceebf24838", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b6a9d785c5d1fbd18d3f9f7130ce8a9f3b95ec3e14ba0f1a5ae557f50b2b58a", + "zh:be00333ae3bbb520d205d8dcf7ddb84fad60cbe0104b6f45bb1fa8d37ad1fbfb", + "zh:cce05ea51cb86ca5cc4bd11a6f9d7c0463cdf74bc45bccbdcbc2b3a6c6465248", + "zh:d5d065af722ad73874f952c0a0d75a35e7c58bceba153b2411e869ad41ff0aa8", + "zh:e8e7abadccca7f484f4e8bb4d2639ba3e89efa94f1d456eb5f173f7053127d63", + "zh:f6b9d9947026ea4e7bb9a2e7c4f15a57dbf26d7b3799df24405a3e7bc0d9ea00", + "zh:ff0cfd9c52c724aef6ffda20285b7fb77b6b044f39308a95dc4f993993802bb4", ] } diff --git a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl b/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl index 61c54c45c8..4717367a16 100644 --- a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl +++ b/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl @@ -21,20 +21,21 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.1.0" + version = "3.1.1" hashes = [ - "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", - "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", - "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", - "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", - "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", - "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", - "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", - "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", - "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", - "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", - "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", - "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + "h1:wK4ig6D02sQtpYs82HNJ8RnYeMcfz+eQMjesc+b4QeE=", + "zh:04dbe3a562a5da087aca7729332dc4bdba763e71a7a9fcf2b71d38454af8975d", + "zh:23ad79fbd96be6271560cb78855b51129845a1a52edff19f6725f99851e36367", + "zh:34678e44716d76bfad398506757680b3285bf8b2704e0637fcec9c3f3b659f85", + "zh:46c64dfa7c7c48576240cda9a022e84e4a4bd049d22bcb8e5a72b9ceebf24838", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b6a9d785c5d1fbd18d3f9f7130ce8a9f3b95ec3e14ba0f1a5ae557f50b2b58a", + "zh:be00333ae3bbb520d205d8dcf7ddb84fad60cbe0104b6f45bb1fa8d37ad1fbfb", + "zh:cce05ea51cb86ca5cc4bd11a6f9d7c0463cdf74bc45bccbdcbc2b3a6c6465248", + "zh:d5d065af722ad73874f952c0a0d75a35e7c58bceba153b2411e869ad41ff0aa8", + "zh:e8e7abadccca7f484f4e8bb4d2639ba3e89efa94f1d456eb5f173f7053127d63", + "zh:f6b9d9947026ea4e7bb9a2e7c4f15a57dbf26d7b3799df24405a3e7bc0d9ea00", + "zh:ff0cfd9c52c724aef6ffda20285b7fb77b6b044f39308a95dc4f993993802bb4", ] } diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 6a0c2a98ec..6b86817f6c 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -1,6 +1,7 @@ --- #cloud-config package_upgrade: true +package_update: true apt: sources: docker.list: @@ -17,6 +18,15 @@ packages: - gnupg2 - pass +# create the docker group +groups: + - docker + +# Add default auto created user to docker group +system_info: + default_user: + groups: [docker] + write_files: # a weekly cron job to have docker free disk space - path: /etc/cron.weekly/docker-prune diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus/terraform/data.tf index 9bd6230220..8cdd03dba5 100644 --- a/templates/shared_services/sonatype-nexus/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus/terraform/data.tf @@ -23,3 +23,8 @@ data "azurerm_key_vault" "kv" { name = "kv-${var.tre_id}" resource_group_name = local.core_resource_group_name } + +data "azurerm_storage_account" "nexus" { + name = local.storage_account_name + resource_group_name = local.core_resource_group_name +} diff --git a/templates/shared_services/sonatype-nexus/terraform/locals.tf b/templates/shared_services/sonatype-nexus/terraform/locals.tf index 50a394f2b9..566f1c102c 100644 --- a/templates/shared_services/sonatype-nexus/terraform/locals.tf +++ b/templates/shared_services/sonatype-nexus/terraform/locals.tf @@ -3,4 +3,5 @@ locals { core_resource_group_name = "rg-${var.tre_id}" firewall_name = "fw-${var.tre_id}" nexus_allowed_fqdns_list = distinct(compact(split(",", replace(var.nexus_allowed_fqdns, " ", "")))) + storage_account_name = lower(replace("stg-${var.tre_id}", "-", "")) } diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus/terraform/variables.tf index 9cf111e741..dddb3d0ee7 100644 --- a/templates/shared_services/sonatype-nexus/terraform/variables.tf +++ b/templates/shared_services/sonatype-nexus/terraform/variables.tf @@ -11,5 +11,5 @@ variable "location" { variable "nexus_allowed_fqdns" { type = string description = "comma seperated string of allowed FQDNs for Nexus" - default = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com,repo.anaconda.com,*.docker.com,*.docker.io" + default = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com,repo.anaconda.com,*.docker.com,*.docker.io,keyserver.ubuntu.com,azure.archive.ubuntu.com" } diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index d471e68bf1..e582a4bd21 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -81,6 +81,10 @@ resource "azurerm_linux_virtual_machine" "nexus" { identity { type = "SystemAssigned" } + + boot_diagnostics { + storage_account_uri = data.azurerm_storage_account.nexus.primary_blob_endpoint + } } data "template_cloudinit_config" "nexus_config" { From b51c5f883f1ede042290a33228e64f6dedc8a3fa Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Thu, 17 Mar 2022 11:53:34 +0000 Subject: [PATCH 016/142] Documented Letsencrypt process --- docs/tre-developers/letsencrypt.md | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/tre-developers/letsencrypt.md diff --git a/docs/tre-developers/letsencrypt.md b/docs/tre-developers/letsencrypt.md new file mode 100644 index 0000000000..39e4a906df --- /dev/null +++ b/docs/tre-developers/letsencrypt.md @@ -0,0 +1,52 @@ +# Letsencrypt + +Certain components of the TRE require the aquisition of a certificate via Letsencrypt to ensure secure HTTPS connections. + +In order to aquire these certificates, there must be a public facing endpoint which can be reached by Letsencrypt. + +As TREs are secured environments with very few publicly facing points, additional resources are required ensure the certificate can be provisioned for the correct domain. + +The additional resources are as followed: + +1. Public IP provisioned in the same location as the web app the certificate is intended for with a domain lable which matches the web apps name. +1. Storage Account with a static web app. +1. Application gateway to route traffic from thepPublic IP to the static web app + +The following diagram illustrated the flow of data between the resources: + +```mermaid +flowchart RL + subgraph .dev Container + direction TB + A(letsencrypt process runs) + end + subgraph External + direction TB + B[letsencrypt authority] + end + subgraph TRE + subgraph Core VNet + C[Public IP
Domain Label: < web-app-name >
Endpoint: < web-app-name >.< location >.cloudapp.net] + subgraph Storage Account + D[SA Static Site] + end + end + subgraph VNet + E[Key Vault
kv-< tre_id >] + subgraph VM + F[Web App] + end + G[Private DNS Zone < web-app-name >.< location >.cloudapp.net] + end + end + + A --> |1. Request to | B + B --> |2. Attempts to hit | C + C --> |3. App Gateway routes | D + D --> |4. Respondes | C + C --> |5. Respondes | B + B --> |6. Aquires certificate | A + A --> |7. Stores Certificate | E + F --> |8. Pulls Certificate | E + + ``` From a1ab6de4975f2a41a08bffc65d931261834423ca Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 17 Mar 2022 12:06:14 +0000 Subject: [PATCH 017/142] Permissions fix --- .../shared_services/sonatype-nexus/terraform/cloud-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 6b86817f6c..321c779c68 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -43,6 +43,8 @@ write_files: runcmd: - export DEBIAN_FRONTEND=noninteractive + # Give the Nexus process write permissions on the folder mounted as persistent volume + - chown -R 200 /etc/nexus-data # Run the nexus container (exposing port 8081 and mapping volume for nexus config) - docker run -d -p 8081:8081 From e8e2209db2ee6a9845baf953f13fde95b3735e5b Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Thu, 17 Mar 2022 12:59:42 +0000 Subject: [PATCH 018/142] Typo fix Co-authored-by: Stuart Leeks --- docs/tre-developers/letsencrypt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-developers/letsencrypt.md b/docs/tre-developers/letsencrypt.md index 39e4a906df..19434250ec 100644 --- a/docs/tre-developers/letsencrypt.md +++ b/docs/tre-developers/letsencrypt.md @@ -43,7 +43,7 @@ flowchart RL A --> |1. Request to | B B --> |2. Attempts to hit | C C --> |3. App Gateway routes | D - D --> |4. Respondes | C + D --> |4. Responds | C C --> |5. Respondes | B B --> |6. Aquires certificate | A A --> |7. Stores Certificate | E From 55cdf19fcd42074f6f7c17046ea0b9afe8373b0d Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Thu, 17 Mar 2022 12:59:49 +0000 Subject: [PATCH 019/142] Typo fix Co-authored-by: Stuart Leeks --- docs/tre-developers/letsencrypt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-developers/letsencrypt.md b/docs/tre-developers/letsencrypt.md index 19434250ec..fd80cb63ae 100644 --- a/docs/tre-developers/letsencrypt.md +++ b/docs/tre-developers/letsencrypt.md @@ -44,7 +44,7 @@ flowchart RL B --> |2. Attempts to hit | C C --> |3. App Gateway routes | D D --> |4. Responds | C - C --> |5. Respondes | B + C --> |5. Responds | B B --> |6. Aquires certificate | A A --> |7. Stores Certificate | E F --> |8. Pulls Certificate | E From a1748d671f67644913dbe2aef23f49e743e9d815 Mon Sep 17 00:00:00 2001 From: Oliver Atkinson Date: Thu, 17 Mar 2022 13:00:00 +0000 Subject: [PATCH 020/142] Typo fix Co-authored-by: Stuart Leeks --- docs/tre-developers/letsencrypt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-developers/letsencrypt.md b/docs/tre-developers/letsencrypt.md index fd80cb63ae..41f4b5d73c 100644 --- a/docs/tre-developers/letsencrypt.md +++ b/docs/tre-developers/letsencrypt.md @@ -45,7 +45,7 @@ flowchart RL C --> |3. App Gateway routes | D D --> |4. Responds | C C --> |5. Responds | B - B --> |6. Aquires certificate | A + B --> |6. Acquires certificate | A A --> |7. Stores Certificate | E F --> |8. Pulls Certificate | E From 5616e03ebb3ec0b581e3d1d193774049101b561c Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Thu, 17 Mar 2022 13:01:14 +0000 Subject: [PATCH 021/142] Formatting changes --- docs/tre-developers/letsencrypt.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tre-developers/letsencrypt.md b/docs/tre-developers/letsencrypt.md index 41f4b5d73c..5fdd4da911 100644 --- a/docs/tre-developers/letsencrypt.md +++ b/docs/tre-developers/letsencrypt.md @@ -43,9 +43,9 @@ flowchart RL A --> |1. Request to | B B --> |2. Attempts to hit | C C --> |3. App Gateway routes | D - D --> |4. Responds | C - C --> |5. Responds | B - B --> |6. Acquires certificate | A + D --> |4. Responds | C + C --> |5. Responds | B + B --> |6. Acquires certificate | A A --> |7. Stores Certificate | E F --> |8. Pulls Certificate | E From f42e96bceb1aaea1220b0e2fef163edbd5a312e0 Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Thu, 17 Mar 2022 13:07:28 +0000 Subject: [PATCH 022/142] Added reference to letsencrypt doc --- templates/core/terraform/appgateway/staticweb.tf | 1 + templates/shared_services/nexus-cert/terraform/staticweb.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/templates/core/terraform/appgateway/staticweb.tf b/templates/core/terraform/appgateway/staticweb.tf index 7cc7d91c31..79f9bdcbc9 100644 --- a/templates/core/terraform/appgateway/staticweb.tf +++ b/templates/core/terraform/appgateway/staticweb.tf @@ -1,5 +1,6 @@ data "azurerm_client_config" "deployer" {} +# See https://microsoft.github.io/AzureTRE/tre-developers/letsencrypt/ resource "azurerm_storage_account" "staticweb" { name = local.staticweb_storage_name resource_group_name = var.resource_group_name diff --git a/templates/shared_services/nexus-cert/terraform/staticweb.tf b/templates/shared_services/nexus-cert/terraform/staticweb.tf index fdd4aab6f6..e5fc7829dc 100644 --- a/templates/shared_services/nexus-cert/terraform/staticweb.tf +++ b/templates/shared_services/nexus-cert/terraform/staticweb.tf @@ -1,5 +1,6 @@ data "azurerm_client_config" "deployer" {} +# See https://microsoft.github.io/AzureTRE/tre-developers/letsencrypt/ resource "azurerm_storage_account" "staticweb" { name = local.staticweb_storage_name resource_group_name = local.core_resource_group_name From 5f30077eda22e638180348bf10fe3bb1cba4eb98 Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Thu, 17 Mar 2022 13:09:20 +0000 Subject: [PATCH 023/142] Added new page reference --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index a206290326..75d08e7ad8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -65,6 +65,7 @@ nav: - API: 'tre-developers/api.md' - Resource Processor: 'tre-developers/resource-processor.md' - End to End Tests: 'tre-developers/end-to-end-tests.md' + - Letsencrypt: 'tre-developers/letsencrypt.md' - TRE Workspace Authors: - Authoring Workspace Templates: 'tre-workspace-authors/authoring-workspace-templates.md' - Firewall Rules: 'tre-workspace-authors/firewall-rules.md' From 126cacdebb61b227776b17031ad35036d3714a84 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 17 Mar 2022 16:28:03 +0000 Subject: [PATCH 024/142] Moved password generation for nexus to tf --- .../sonatype-nexus/scripts/configure_nexus.sh | 44 +++++++------------ .../terraform/cloud-config.yaml | 4 ++ .../sonatype-nexus/terraform/vm.tf | 28 ++++++++++-- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index 25909da9bd..13f2b2a51f 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -4,10 +4,11 @@ set -e function usage() { cat < Date: Mon, 21 Mar 2022 16:33:05 +0000 Subject: [PATCH 025/142] Write script to fs first before execution --- templates/core/terraform/.terraform.lock.hcl | 26 ++++++------- .../gitea/terraform/.terraform.lock.hcl | 26 ++++++------- .../sonatype-nexus/scripts/configure_nexus.sh | 6 +-- .../terraform/.terraform.lock.hcl | 26 ++++++------- .../terraform/cloud-config.yaml | 39 ++++++++++++++----- .../sonatype-nexus/terraform/vm.tf | 2 +- 6 files changed, 71 insertions(+), 54 deletions(-) diff --git a/templates/core/terraform/.terraform.lock.hcl b/templates/core/terraform/.terraform.lock.hcl index 3505fcea5d..21a46d25a4 100644 --- a/templates/core/terraform/.terraform.lock.hcl +++ b/templates/core/terraform/.terraform.lock.hcl @@ -58,21 +58,21 @@ provider "registry.terraform.io/hashicorp/local" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.1.1" + version = "3.1.2" hashes = [ - "h1:wK4ig6D02sQtpYs82HNJ8RnYeMcfz+eQMjesc+b4QeE=", - "zh:04dbe3a562a5da087aca7729332dc4bdba763e71a7a9fcf2b71d38454af8975d", - "zh:23ad79fbd96be6271560cb78855b51129845a1a52edff19f6725f99851e36367", - "zh:34678e44716d76bfad398506757680b3285bf8b2704e0637fcec9c3f3b659f85", - "zh:46c64dfa7c7c48576240cda9a022e84e4a4bd049d22bcb8e5a72b9ceebf24838", + "h1:5A5VsY5wNmOZlupUcLnIoziMPn8htSZBXbP3lI7lBEM=", + "zh:0daceba867b330d3f8e2c5dc895c4291845a78f31955ce1b91ab2c4d1cd1c10b", + "zh:104050099efd30a630741f788f9576b19998e7a09347decbec3da0b21d64ba2d", + "zh:173f4ef3fdf0c7e2564a3db0fac560e9f5afdf6afd0b75d6646af6576b122b16", + "zh:41d50f975e535f968b3f37170fb07937c15b76d85ba947d0ce5e5ff9530eda65", + "zh:51a5038867e5e60757ed7f513dd6a973068241190d158a81d1b69296efb9cb8d", + "zh:6432a568e97a5a36cc8aebca5a7e9c879a55d3bc71d0da1ab849ad905f41c0be", + "zh:6bac6501394b87138a5e17c9f3a41e46ff7833ad0ba2a96197bb7787e95b641c", + "zh:6c0a7f5faacda644b022e7718e53f5868187435be6d000786d1ca05aa6683a25", + "zh:74c89de3fa6ef3027efe08f8473c2baeb41b4c6cee250ba7aeb5b64e8c79800d", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b6a9d785c5d1fbd18d3f9f7130ce8a9f3b95ec3e14ba0f1a5ae557f50b2b58a", - "zh:be00333ae3bbb520d205d8dcf7ddb84fad60cbe0104b6f45bb1fa8d37ad1fbfb", - "zh:cce05ea51cb86ca5cc4bd11a6f9d7c0463cdf74bc45bccbdcbc2b3a6c6465248", - "zh:d5d065af722ad73874f952c0a0d75a35e7c58bceba153b2411e869ad41ff0aa8", - "zh:e8e7abadccca7f484f4e8bb4d2639ba3e89efa94f1d456eb5f173f7053127d63", - "zh:f6b9d9947026ea4e7bb9a2e7c4f15a57dbf26d7b3799df24405a3e7bc0d9ea00", - "zh:ff0cfd9c52c724aef6ffda20285b7fb77b6b044f39308a95dc4f993993802bb4", + "zh:b29eabbf0a5298f0e95a1df214c7cfe06ea9bcf362c63b3ad2f72d85da7d4685", + "zh:e891458c7a61e5b964e09616f1a4f87d0471feae1ec04cc51776e7dec1a3abce", ] } diff --git a/templates/shared_services/gitea/terraform/.terraform.lock.hcl b/templates/shared_services/gitea/terraform/.terraform.lock.hcl index 11ffd4227a..628d25f93f 100644 --- a/templates/shared_services/gitea/terraform/.terraform.lock.hcl +++ b/templates/shared_services/gitea/terraform/.terraform.lock.hcl @@ -40,20 +40,20 @@ provider "registry.terraform.io/hashicorp/local" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.1.1" + version = "3.1.2" hashes = [ - "h1:wK4ig6D02sQtpYs82HNJ8RnYeMcfz+eQMjesc+b4QeE=", - "zh:04dbe3a562a5da087aca7729332dc4bdba763e71a7a9fcf2b71d38454af8975d", - "zh:23ad79fbd96be6271560cb78855b51129845a1a52edff19f6725f99851e36367", - "zh:34678e44716d76bfad398506757680b3285bf8b2704e0637fcec9c3f3b659f85", - "zh:46c64dfa7c7c48576240cda9a022e84e4a4bd049d22bcb8e5a72b9ceebf24838", + "h1:5A5VsY5wNmOZlupUcLnIoziMPn8htSZBXbP3lI7lBEM=", + "zh:0daceba867b330d3f8e2c5dc895c4291845a78f31955ce1b91ab2c4d1cd1c10b", + "zh:104050099efd30a630741f788f9576b19998e7a09347decbec3da0b21d64ba2d", + "zh:173f4ef3fdf0c7e2564a3db0fac560e9f5afdf6afd0b75d6646af6576b122b16", + "zh:41d50f975e535f968b3f37170fb07937c15b76d85ba947d0ce5e5ff9530eda65", + "zh:51a5038867e5e60757ed7f513dd6a973068241190d158a81d1b69296efb9cb8d", + "zh:6432a568e97a5a36cc8aebca5a7e9c879a55d3bc71d0da1ab849ad905f41c0be", + "zh:6bac6501394b87138a5e17c9f3a41e46ff7833ad0ba2a96197bb7787e95b641c", + "zh:6c0a7f5faacda644b022e7718e53f5868187435be6d000786d1ca05aa6683a25", + "zh:74c89de3fa6ef3027efe08f8473c2baeb41b4c6cee250ba7aeb5b64e8c79800d", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b6a9d785c5d1fbd18d3f9f7130ce8a9f3b95ec3e14ba0f1a5ae557f50b2b58a", - "zh:be00333ae3bbb520d205d8dcf7ddb84fad60cbe0104b6f45bb1fa8d37ad1fbfb", - "zh:cce05ea51cb86ca5cc4bd11a6f9d7c0463cdf74bc45bccbdcbc2b3a6c6465248", - "zh:d5d065af722ad73874f952c0a0d75a35e7c58bceba153b2411e869ad41ff0aa8", - "zh:e8e7abadccca7f484f4e8bb4d2639ba3e89efa94f1d456eb5f173f7053127d63", - "zh:f6b9d9947026ea4e7bb9a2e7c4f15a57dbf26d7b3799df24405a3e7bc0d9ea00", - "zh:ff0cfd9c52c724aef6ffda20285b7fb77b6b044f39308a95dc4f993993802bb4", + "zh:b29eabbf0a5298f0e95a1df214c7cfe06ea9bcf362c63b3ad2f72d85da7d4685", + "zh:e891458c7a61e5b964e09616f1a4f87d0471feae1ec04cc51776e7dec1a3abce", ] } diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index 13f2b2a51f..4b19e460a9 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -38,15 +38,13 @@ while [ "$1" != "" ]; do shift # remove the current value for `$1` and use the next done -export NEXUS_URL="https://nexus-${tre_id}.${location}.cloudapp.azure.com:8081" +export NEXUS_URL="http://nexus-${tre_id}.${location}.cloudapp.azure.com:8081" # TODO: change to https once ssl cert is added export NEXUS_ADMIN_PASSWORD_NAME="nexus-admin-password" export KEYVAULT_NAME="kv-${tre_id}" -export STORAGE_ACCOUNT_NAME="stg${tre_id//-/}" - export NEXUS_PASS=$(az keyvault secret show --name ${NEXUS_ADMIN_PASSWORD_NAME} --vault-name ${KEYVAULT_NAME} -o json | jq -r '.value') if [ -z "$NEXUS_PASS" ]; then - echo "Unable to get the Nexus admin password from Keyvault. You may need to manually reset it - refer to the public Nexus documentation for more information." + echo "Unable to get the Nexus admin password from Keyvault. You may need to manually reset it in the Nexus host. Refer to the public Nexus documentation for more information." exit 1 fi diff --git a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl b/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl index 4717367a16..0051ace45a 100644 --- a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl +++ b/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl @@ -21,21 +21,21 @@ provider "registry.terraform.io/hashicorp/azurerm" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.1.1" + version = "3.1.2" hashes = [ - "h1:wK4ig6D02sQtpYs82HNJ8RnYeMcfz+eQMjesc+b4QeE=", - "zh:04dbe3a562a5da087aca7729332dc4bdba763e71a7a9fcf2b71d38454af8975d", - "zh:23ad79fbd96be6271560cb78855b51129845a1a52edff19f6725f99851e36367", - "zh:34678e44716d76bfad398506757680b3285bf8b2704e0637fcec9c3f3b659f85", - "zh:46c64dfa7c7c48576240cda9a022e84e4a4bd049d22bcb8e5a72b9ceebf24838", + "h1:5A5VsY5wNmOZlupUcLnIoziMPn8htSZBXbP3lI7lBEM=", + "zh:0daceba867b330d3f8e2c5dc895c4291845a78f31955ce1b91ab2c4d1cd1c10b", + "zh:104050099efd30a630741f788f9576b19998e7a09347decbec3da0b21d64ba2d", + "zh:173f4ef3fdf0c7e2564a3db0fac560e9f5afdf6afd0b75d6646af6576b122b16", + "zh:41d50f975e535f968b3f37170fb07937c15b76d85ba947d0ce5e5ff9530eda65", + "zh:51a5038867e5e60757ed7f513dd6a973068241190d158a81d1b69296efb9cb8d", + "zh:6432a568e97a5a36cc8aebca5a7e9c879a55d3bc71d0da1ab849ad905f41c0be", + "zh:6bac6501394b87138a5e17c9f3a41e46ff7833ad0ba2a96197bb7787e95b641c", + "zh:6c0a7f5faacda644b022e7718e53f5868187435be6d000786d1ca05aa6683a25", + "zh:74c89de3fa6ef3027efe08f8473c2baeb41b4c6cee250ba7aeb5b64e8c79800d", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b6a9d785c5d1fbd18d3f9f7130ce8a9f3b95ec3e14ba0f1a5ae557f50b2b58a", - "zh:be00333ae3bbb520d205d8dcf7ddb84fad60cbe0104b6f45bb1fa8d37ad1fbfb", - "zh:cce05ea51cb86ca5cc4bd11a6f9d7c0463cdf74bc45bccbdcbc2b3a6c6465248", - "zh:d5d065af722ad73874f952c0a0d75a35e7c58bceba153b2411e869ad41ff0aa8", - "zh:e8e7abadccca7f484f4e8bb4d2639ba3e89efa94f1d456eb5f173f7053127d63", - "zh:f6b9d9947026ea4e7bb9a2e7c4f15a57dbf26d7b3799df24405a3e7bc0d9ea00", - "zh:ff0cfd9c52c724aef6ffda20285b7fb77b6b044f39308a95dc4f993993802bb4", + "zh:b29eabbf0a5298f0e95a1df214c7cfe06ea9bcf362c63b3ad2f72d85da7d4685", + "zh:e891458c7a61e5b964e09616f1a4f87d0471feae1ec04cc51776e7dec1a3abce", ] } diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 179f3197d6..694f7d49de 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -40,6 +40,27 @@ write_files: content: | nexus.skipDefaultRepositories=true permissions: '0755' + # Write a script that will reset the admin password for Nexus to the one TF generated + - path: /home/adminuser/reset-nexus-password.sh + content: | + #!/bin/bash + # Get the current password so we can post to the API + # (this is created in the /nexus-data mounted volume as part of Nexus container start-up in cloud-init) + timeout=120 + while [ ! -f /etc/nexus-data/admin.password ]; do + # We must first wait for the file to be created + if [ $timeout == 0 ]; then + echo 'ERROR - Timeout while waiting for nexus-data/admin.password to be created' + exit 1 + fi + sleep 1 + ((timeout--)) + done + CURRENT_PASSWORD=$(cat /etc/nexus-data/admin.password) + # Set our own admin password so we can connect to the Nexus repository manager later on using TF KV secret + curl -ifu admin:$CURRENT_PASSWORD -XPUT -H 'Content-Type:text/plain' --data '${nexus_admin_password}' \ + http://localhost:8081/service/rest/v1/security/users/admin/change-password + permissions: '0744' runcmd: - export DEBIAN_FRONTEND=noninteractive @@ -47,13 +68,11 @@ runcmd: - chown -R 200 /etc/nexus-data # Run the nexus container (exposing port 8081 and mapping volume for nexus config) - docker run -d - -p 8081:8081 - -v /etc/nexus-data:/nexus-data - --restart always - --name nexus - --log-driver local - sonatype/nexus3 - # Set our own admin password so we can connect to the Nexus repository manager later on - - CURRENT_PASSWORD=$(cat /etc/nexus-data/admin.password) - - curl -ifu admin:${CURRENT_PASSWORD} -XPUT -H 'Content-Type: text/plain' --data "${nexus_admin_password}" \ - http://localhost:8081/service/rest/v1/security/users/admin/change-password + -p 8081:8081 + -v /etc/nexus-data:/nexus-data + --restart always + --name nexus + --log-driver local + sonatype/nexus3 + # Run the script to reset Nexus password (we write this as a script file first to avoid tricky handling of special chars in yml) + - bash /home/adminuser/reset-nexus-password.sh diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 71133870eb..db49e053e6 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -64,7 +64,7 @@ resource "azurerm_key_vault_secret" "nexus_vm_password" { key_vault_id = data.azurerm_key_vault.kv.id } -resource "azurerm_key_vault_secret" "nexus_vm_password" { +resource "azurerm_key_vault_secret" "nexus_admin_password" { name = "nexus-admin-password" value = random_password.nexus_admin_password.result key_vault_id = data.azurerm_key_vault.kv.id From c26aa9fefc01f08e627c12b952f7387a1a670648 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 21 Mar 2022 21:35:34 +0000 Subject: [PATCH 026/142] Password reset finally working --- .../sonatype-nexus/scripts/configure_nexus.sh | 2 +- .../terraform/cloud-config.yaml | 33 +++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index 4b19e460a9..4a06279058 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -38,7 +38,7 @@ while [ "$1" != "" ]; do shift # remove the current value for `$1` and use the next done -export NEXUS_URL="http://nexus-${tre_id}.${location}.cloudapp.azure.com:8081" # TODO: change to https once ssl cert is added +export NEXUS_URL="http://nexus-${tre_id}.${location}.cloudapp.azure.com" # TODO: change to https once ssl cert is added export NEXUS_ADMIN_PASSWORD_NAME="nexus-admin-password" export KEYVAULT_NAME="kv-${tre_id}" export NEXUS_PASS=$(az keyvault secret show --name ${NEXUS_ADMIN_PASSWORD_NAME} --vault-name ${KEYVAULT_NAME} -o json | jq -r '.value') diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 694f7d49de..c946be04c3 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -46,20 +46,41 @@ write_files: #!/bin/bash # Get the current password so we can post to the API # (this is created in the /nexus-data mounted volume as part of Nexus container start-up in cloud-init) - timeout=120 + password_timeout=120 + echo 'Checking for Nexus admin password file...' while [ ! -f /etc/nexus-data/admin.password ]; do # We must first wait for the file to be created - if [ $timeout == 0 ]; then + if [ $password_timeout == 0 ]; then echo 'ERROR - Timeout while waiting for nexus-data/admin.password to be created' exit 1 fi sleep 1 - ((timeout--)) + ((password_timeout--)) done CURRENT_PASSWORD=$(cat /etc/nexus-data/admin.password) + # Set our own admin password so we can connect to the Nexus repository manager later on using TF KV secret - curl -ifu admin:$CURRENT_PASSWORD -XPUT -H 'Content-Type:text/plain' --data '${nexus_admin_password}' \ - http://localhost:8081/service/rest/v1/security/users/admin/change-password + reset_timeout=120 + echo 'Nexus default admin password found. Resetting...' + # While the container is starting up it may return a number of transient errors (i.e. connection reset) which we need to retry until operational + # NOTE: we can't use curl's built-in retry flags as it doesn't catch for the connection reset response + res=1 + while test "$res" != "0"; do + curl -ifu admin:$CURRENT_PASSWORD -XPUT -H 'Content-Type:text/plain' --data '${nexus_admin_password}' \ + http://localhost/service/rest/v1/security/users/admin/change-password + $res=$? + echo "Attempt to reset password finished with code $res" + if test "$res" == "0"; then + echo 'Password reset successfully. Admin can now log in with secret stored in KeyVault. Nexus configuration complete.' + else + if [ $reset_timeout == 0 ]; then + echo 'ERROR - Timeout while trying to reset Nexus admin password' + exit 1 + fi + sleep 5 + ((reset_timeout+=5)) + fi + done permissions: '0744' runcmd: @@ -68,7 +89,7 @@ runcmd: - chown -R 200 /etc/nexus-data # Run the nexus container (exposing port 8081 and mapping volume for nexus config) - docker run -d - -p 8081:8081 + -p 80:8081 -p 443:8081 -v /etc/nexus-data:/nexus-data --restart always --name nexus From 3c45b93092acbe202059bf8e905ba03c32a4a225 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 21 Mar 2022 21:58:17 +0000 Subject: [PATCH 027/142] Make config nexus script runnable from any dir --- .../shared_services/sonatype-nexus/scripts/configure_nexus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index 4a06279058..a0457b8a17 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -49,7 +49,7 @@ if [ -z "$NEXUS_PASS" ]; then fi # Create proxy for each .json file -for filename in ./scripts/nexus_config/*.json; do +for filename in "$(dirname ${BASH_SOURCE[0]})/nexus_config/*.json"; do # Check if apt proxy base_type=$( jq .baseType $filename | sed 's/"//g') repo_type=$( jq .repoType $filename | sed 's/"//g') From 7fee96ba3525f710b1d8a58d36bd6cb70e70e1f7 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 21 Mar 2022 22:02:48 +0000 Subject: [PATCH 028/142] Added basic status info --- .../shared_services/sonatype-nexus/scripts/configure_nexus.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index a0457b8a17..d382a1aabd 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -50,6 +50,7 @@ fi # Create proxy for each .json file for filename in "$(dirname ${BASH_SOURCE[0]})/nexus_config/*.json"; do + echo "Found config file: $filename. Sending to Nexus..." # Check if apt proxy base_type=$( jq .baseType $filename | sed 's/"//g') repo_type=$( jq .repoType $filename | sed 's/"//g') @@ -59,6 +60,7 @@ for filename in "$(dirname ${BASH_SOURCE[0]})/nexus_config/*.json"; do full_url=$base_url/$repo_name export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" $full_url -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) + echo "Response received from Nexus: $STATUS_CODE" if [[ ${STATUS_CODE} == 404 ]] then From 73ff598b452747f0ecc175c8624ca980a2bf713f Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 21 Mar 2022 22:08:51 +0000 Subject: [PATCH 029/142] Fix recursive file loop --- .../shared_services/sonatype-nexus/scripts/configure_nexus.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index d382a1aabd..054ecf49e5 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -49,7 +49,7 @@ if [ -z "$NEXUS_PASS" ]; then fi # Create proxy for each .json file -for filename in "$(dirname ${BASH_SOURCE[0]})/nexus_config/*.json"; do +for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_config/*.json; do echo "Found config file: $filename. Sending to Nexus..." # Check if apt proxy base_type=$( jq .baseType $filename | sed 's/"//g') @@ -67,7 +67,7 @@ for filename in "$(dirname ${BASH_SOURCE[0]})/nexus_config/*.json"; do curl -iu admin:$NEXUS_PASS -XPOST \ $base_url \ -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ + -H 'Content-Type: application/json' \cd -d @$filename else echo "$repo_type proxy for $repo_name already exists." From e38e69eec034663c044f81bc3101380c8b37b2e4 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 21 Mar 2022 22:12:51 +0000 Subject: [PATCH 030/142] Typo fix --- .../shared_services/sonatype-nexus/scripts/configure_nexus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index 054ecf49e5..d52556fc29 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -67,7 +67,7 @@ for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_config/*.json; do curl -iu admin:$NEXUS_PASS -XPOST \ $base_url \ -H 'accept: application/json' \ - -H 'Content-Type: application/json' \cd + -H 'Content-Type: application/json' \ -d @$filename else echo "$repo_type proxy for $repo_name already exists." From fbf01c29d02888ec7b73b9410f2244b5a8cbbb7a Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 22 Mar 2022 11:21:26 +0000 Subject: [PATCH 031/142] Updated docs --- .../setup-instructions/configuring-shared-services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 4e0709003f..99a876b1f6 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -15,7 +15,7 @@ Complete the configuration of the shared services (Nexus and Gitea) from inside ## Configure Nexus repository 1. Run the Nexus configuration script to reset the password and setup a PyPI proxy on Nexus: -```./templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh -t ``` +```./templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh -t -l ``` ## Configure Gitea repository From 1599146b375fac0eec2d1516a521809bad5babbc Mon Sep 17 00:00:00 2001 From: oliver7598 Date: Tue, 22 Mar 2022 15:08:29 +0000 Subject: [PATCH 032/142] renamed env file --- Makefile | 2 +- templates/shared_services/nexus-cert/scripts/outputs.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43458384d0..5e7df165f2 100644 --- a/Makefile +++ b/Makefile @@ -157,7 +157,7 @@ nexus-letsencrypt: && . ./devops/scripts/load_terraform_env.sh ./devops/.env \ && . ./devops/scripts/load_terraform_env.sh ./templates/core/.env \ && pushd ./templates/shared_services/nexus-cert/scripts/ > /dev/null && . ./outputs.sh && popd > /dev/null \ - && . ./devops/scripts/load_env.sh ./templates/shared_services/nexus-cert/private.env \ + && . ./devops/scripts/load_env.sh ./templates/shared_services/nexus-cert/.env \ && ./templates/shared_services/nexus-cert/scripts/letsencrypt.sh deploy-core: tre-start diff --git a/templates/shared_services/nexus-cert/scripts/outputs.sh b/templates/shared_services/nexus-cert/scripts/outputs.sh index 5a94968ba1..04d21c9452 100644 --- a/templates/shared_services/nexus-cert/scripts/outputs.sh +++ b/templates/shared_services/nexus-cert/scripts/outputs.sh @@ -15,4 +15,4 @@ if [ ! -f ../tre_output.json ]; then fi # Now create an .env file -./json-to-env.sh < ../tre_output.json > ../private.env +./json-to-env.sh < ../tre_output.json > ../.env From 220954b608ffc8060a049de0d22a348de8402e31 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 22 Mar 2022 15:49:49 +0000 Subject: [PATCH 033/142] Fix typo --- .../guacamole-azure-windowsvm/terraform/vm_config.ps1 | 2 +- .../guacamole-azure-windowsvm/terraform/windowsvm.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 index 58eb9037ec..4b74c0e0d7 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 @@ -30,4 +30,4 @@ if( ${CondaConfig} -eq 1 ) conda config --add channels ${nexus_proxy_url}/repository/conda/ --system conda config --remove channels defaults --system conda config --set channel_alias ${nexus_proxy_url}/repository/conda/ --system -} \ No newline at end of file +}cd .. diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf index 5e4fe3c5a0..8b443d3f0c 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf @@ -68,7 +68,7 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { } resource "azurerm_virtual_machine_extension" "config_script" { - name = "${azurerm_windows_virtual_machine.windowsvm.name}-vmextention" + name = "${azurerm_windows_virtual_machine.windowsvm.name}-vmextension" virtual_machine_id = azurerm_windows_virtual_machine.windowsvm.id publisher = "Microsoft.Compute" type = "CustomScriptExtension" From ba320c83f9ec0dc34cc5d77b32e3c4bbde5a290c Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 23 Mar 2022 11:28:38 +0000 Subject: [PATCH 034/142] Added new nexus fqdn to user resources --- resource_processor/shared/config.py | 1 + templates/core/terraform/json-to-env.sh | 4 ++++ templates/core/terraform/outputs.tf | 4 ++++ .../resource_processor/vmss_porter/cloud-config.yaml | 1 + .../core/terraform/resource_processor/vmss_porter/main.tf | 1 + .../user_resources/guacamole-azure-linuxvm/porter.yaml | 4 ++++ .../guacamole-azure-linuxvm/terraform/locals.tf | 2 +- .../guacamole-azure-linuxvm/terraform/variables.tf | 1 + .../user_resources/guacamole-azure-windowsvm/porter.yaml | 4 ++++ .../guacamole-azure-windowsvm/terraform/locals.tf | 2 +- .../guacamole-azure-windowsvm/terraform/variables.tf | 1 + 11 files changed, 23 insertions(+), 2 deletions(-) diff --git a/resource_processor/shared/config.py b/resource_processor/shared/config.py index 53d0350ead..a474e4cc56 100644 --- a/resource_processor/shared/config.py +++ b/resource_processor/shared/config.py @@ -11,6 +11,7 @@ def get_config() -> dict: config["deployment_status_queue"] = os.environ["SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE"] config["resource_request_queue"] = os.environ["SERVICE_BUS_RESOURCE_REQUEST_QUEUE"] config["service_bus_namespace"] = os.environ["SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE"] + config["location"] = os.environ["LOCATION"] config["vmss_msi_id"] = os.environ.get("VMSS_MSI_ID", None) # Needed for running porter diff --git a/templates/core/terraform/json-to-env.sh b/templates/core/terraform/json-to-env.sh index 57b797b5a9..d88c89ca95 100755 --- a/templates/core/terraform/json-to-env.sh +++ b/templates/core/terraform/json-to-env.sh @@ -76,6 +76,10 @@ jq -r ' { "path": "registry_server", "env_var": "REGISTRY_SERVER" + }, + { + "path": "location", + "env_var": "LOCATION" } ] as $env_vars_to_extract diff --git a/templates/core/terraform/outputs.tf b/templates/core/terraform/outputs.tf index 00ff260525..6ccfa30b10 100644 --- a/templates/core/terraform/outputs.tf +++ b/templates/core/terraform/outputs.tf @@ -76,3 +76,7 @@ output "terraform_state_container_name" { output "registry_server" { value = var.docker_registry_server } + +output "location" { + value = var.location +} diff --git a/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml b/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml index d96177367f..6cb3ffeeae 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml +++ b/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml @@ -39,6 +39,7 @@ write_files: AZURE_TENANT_ID=${arm_tenant_id} ARM_USE_MSI=true APPLICATIONINSIGHTS_CONNECTION_STRING=${app_insights_connection_string} + LOCATION=${location} # a weekly cron job to have docker free disk space - path: /etc/cron.weekly/docker-prune content: | diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index ccabe0bfe2..4f2de301ed 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -18,6 +18,7 @@ data "template_file" "cloudconfig" { resource_processor_vmss_porter_image_repository = var.resource_processor_vmss_porter_image_repository resource_processor_vmss_porter_image_tag = local.version app_insights_connection_string = var.app_insights_connection_string + location = var.location } } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index b19443dff8..75034ca074 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -30,6 +30,9 @@ parameters: type: string description: "An Id for this installation" env: id + - name: location + type: string + description: "The location the TRE is deployed in" - name: tfstate_resource_group_name type: string description: "Resource group containing the Terraform state storage account" @@ -85,6 +88,7 @@ install: vars: workspace_id: "{{ bundle.parameters.workspace_id }}" tre_id: "{{ bundle.parameters.tre_id }}" + location: "{{ bundle.parameters.location }}" parent_service_id: "{{ bundle.parameters.parent_service_id }}" arm_client_id: "{{ bundle.credentials.azure_client_id }}" arm_client_secret: "{{ bundle.credentials.azure_client_secret }}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf index 9924692707..261b61a665 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf @@ -9,7 +9,7 @@ locals { vm_name = "linuxvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") storage_name = lower(replace("stg${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", "")) - nexus_proxy_url = "https://nexus-${var.tre_id}.azurewebsites.net" + nexus_proxy_url = "https://nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" image_ref = { "Ubuntu 18.04" = { "publisher" = "canonical" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf index bdd7610a7f..45ccde2889 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf @@ -1,6 +1,7 @@ variable "workspace_id" {} variable "tre_id" {} variable "parent_service_id" {} +variable "location" {} variable "arm_client_id" {} variable "arm_client_secret" {} diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index a925a502e0..e426bda87b 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -30,6 +30,9 @@ parameters: type: string description: "An Id for this installation" env: id + - name: location + type: string + description: "The location the TRE is deployed in" - name: tfstate_resource_group_name type: string description: "Resource group containing the Terraform state storage account" @@ -85,6 +88,7 @@ install: vars: workspace_id: "{{ bundle.parameters.workspace_id }}" tre_id: "{{ bundle.parameters.tre_id }}" + location: "{{ bundle.parameters.location }}" parent_service_id: "{{ bundle.parameters.parent_service_id }}" arm_client_id: "{{ bundle.credentials.azure_client_id }}" arm_client_secret: "{{ bundle.credentials.azure_client_secret }}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf index 0c2d5de792..a2ee3d3a83 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf @@ -9,7 +9,7 @@ locals { vm_name = "windowsvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") storage_name = lower(replace("stg${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", "")) - nexus_proxy_url = "https://nexus-${var.tre_id}.azurewebsites.net" + nexus_proxy_url = "https://nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" image_ref = { "Windows 10" = { "publisher" = "MicrosoftWindowsDesktop" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf index bdd7610a7f..45ccde2889 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf @@ -1,6 +1,7 @@ variable "workspace_id" {} variable "tre_id" {} variable "parent_service_id" {} +variable "location" {} variable "arm_client_id" {} variable "arm_client_secret" {} From fc09ab91b2fff8772dae0d2b989a1b4d6a33b580 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 23 Mar 2022 18:00:48 +0000 Subject: [PATCH 035/142] Add vnet link to workspaces --- .../sonatype-nexus/terraform/firewall.tf | 4 ++-- .../shared_services/sonatype-nexus/terraform/vm.tf | 6 +++--- .../guacamole-azure-linuxvm/porter.yaml | 2 +- .../guacamole-azure-windowsvm/porter.yaml | 2 +- templates/workspaces/base/terraform/network.tf | 14 ++++++++++++++ 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/firewall.tf b/templates/shared_services/sonatype-nexus/terraform/firewall.tf index 7649726d82..092f7c3a43 100644 --- a/templates/shared_services/sonatype-nexus/terraform/firewall.tf +++ b/templates/shared_services/sonatype-nexus/terraform/firewall.tf @@ -1,5 +1,5 @@ -resource "azurerm_firewall_application_rule_collection" "vm_subnet_nexus" { - name = "vm_subnet_nexus" +resource "azurerm_firewall_application_rule_collection" "shared_subnet_nexus" { + name = "shared_subnet_nexus" azure_firewall_name = data.azurerm_firewall.fw.name resource_group_name = data.azurerm_firewall.fw.resource_group_name priority = 104 diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index db49e053e6..49a7a4a8e0 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -1,5 +1,5 @@ resource "azurerm_network_interface" "nexus" { - name = "internal-nic-nexus-${var.tre_id}" + name = "nic-nexus-${var.tre_id}" location = var.location resource_group_name = local.core_resource_group_name @@ -17,8 +17,8 @@ resource "azurerm_private_dns_zone" "nexus" { lifecycle { ignore_changes = [tags] } } -resource "azurerm_private_dns_zone_virtual_network_link" "nexus" { - name = "nexus" +resource "azurerm_private_dns_zone_virtual_network_link" "nexus_core_vnet" { + name = "nexuslink-core" resource_group_name = local.core_resource_group_name private_dns_zone_name = azurerm_private_dns_zone.nexus.name virtual_network_id = data.azurerm_virtual_network.core.id diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index 75034ca074..ebc75ce769 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-linuxvm -version: 0.1.7 +version: 0.1.8 description: "An Azure TRE User Resource Template for Guacamole (Linux)" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index e426bda87b..67e739df5b 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-windowsvm -version: 0.1.5 +version: 0.1.6 description: "An Azure TRE User Resource Template for Guacamole (Windows 10)" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspaces/base/terraform/network.tf b/templates/workspaces/base/terraform/network.tf index 56ecdef379..d8cb8c65bf 100644 --- a/templates/workspaces/base/terraform/network.tf +++ b/templates/workspaces/base/terraform/network.tf @@ -422,3 +422,17 @@ resource "azurerm_private_dns_zone_virtual_network_link" "mysqllink" { lifecycle { ignore_changes = [tags] } } + +data "azurerm_private_dns_zone" "nexus" { + name = "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" + resource_group_name = local.core_resource_group_name +} + +resource "azurerm_private_dns_zone_virtual_network_link" "nexuslink" { + name = "nexuslink-${local.workspace_resource_name_suffix}" + resource_group_name = local.core_resource_group_name + private_dns_zone_name = data.azurerm_private_dns_zone.nexus.name + virtual_network_id = azurerm_virtual_network.ws.id + + lifecycle { ignore_changes = [tags] } +} From 517cdb1e4c58801edc896c37a83101f73b8aa631 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 23 Mar 2022 18:04:34 +0000 Subject: [PATCH 036/142] Bump versions --- resource_processor/version.txt | 2 +- templates/shared_services/sonatype-nexus/porter.yaml | 2 +- templates/workspaces/base/porter.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resource_processor/version.txt b/resource_processor/version.txt index ae7362549b..bbab0242f6 100644 --- a/resource_processor/version.txt +++ b/resource_processor/version.txt @@ -1 +1 @@ -__version__ = "0.1.3" +__version__ = "0.1.4" diff --git a/templates/shared_services/sonatype-nexus/porter.yaml b/templates/shared_services/sonatype-nexus/porter.yaml index 6f2294b15f..9ac6331665 100644 --- a/templates/shared_services/sonatype-nexus/porter.yaml +++ b/templates/shared_services/sonatype-nexus/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-nexus -version: 0.0.1 +version: 0.0.2 description: "A Sonatype Nexus shared service" registry: azuretre credentials: diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index f03aed2f94..cc8642eaf0 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-base -version: 0.1.10 +version: 0.1.11 description: "A base Azure TRE workspace" registry: azuretre From 156768abc32c4d9697d924229e6a7b62dd98cad8 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 24 Mar 2022 00:29:58 +0000 Subject: [PATCH 037/142] Removed nexus properties file --- Makefile | 2 +- .../shared_services/sonatype-nexus/terraform/variables.tf | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 74732e5d22..9e37b69bdc 100644 --- a/Makefile +++ b/Makefile @@ -149,7 +149,7 @@ gitea-install: nexus-install: $(call target_title, "Installing Nexus") \ - && make SHARED_SERVICE_KEY=shared-service-sonatype-nexus TF_VAR_nexus_properties_path=../nexus.properties terraform-shared-service-deploy DIR=./templates/shared_services/sonatype-nexus/terraform + && make SHARED_SERVICE_KEY=shared-service-sonatype-nexus terraform-shared-service-deploy DIR=./templates/shared_services/sonatype-nexus/terraform # / End migration targets diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus/terraform/variables.tf index 2aa927c773..c5af1fe95c 100644 --- a/templates/shared_services/sonatype-nexus/terraform/variables.tf +++ b/templates/shared_services/sonatype-nexus/terraform/variables.tf @@ -13,9 +13,3 @@ variable "nexus_allowed_fqdns" { description = "comma seperated string of allowed FQDNs for Nexus" default = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com,repo.anaconda.com,*.docker.com,*.docker.io,conda.anaconda.org" } - -variable "nexus_properties_path" { - type = string - description = "relative path of nexus properties file" - default = "/cnab/app/nexus.properties" -} From e0516233602be3a72acaccb7d0f9fe3432b0b41a Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 24 Mar 2022 11:35:20 +0000 Subject: [PATCH 038/142] Updated execution permissions --- templates/shared_services/nexus-cert/Dockerfile.tmpl | 0 templates/shared_services/nexus-cert/azure.json | 0 templates/shared_services/nexus-cert/parameters.json | 0 templates/shared_services/nexus-cert/porter.yaml | 0 templates/shared_services/nexus-cert/scripts/auth-hook.sh | 0 templates/shared_services/nexus-cert/scripts/cleanup-hook.sh | 0 templates/shared_services/nexus-cert/scripts/json-to-env.sh | 0 templates/shared_services/nexus-cert/scripts/letsencrypt.sh | 0 templates/shared_services/nexus-cert/scripts/outputs.sh | 0 9 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 templates/shared_services/nexus-cert/Dockerfile.tmpl mode change 100644 => 100755 templates/shared_services/nexus-cert/azure.json mode change 100644 => 100755 templates/shared_services/nexus-cert/parameters.json mode change 100644 => 100755 templates/shared_services/nexus-cert/porter.yaml mode change 100644 => 100755 templates/shared_services/nexus-cert/scripts/auth-hook.sh mode change 100644 => 100755 templates/shared_services/nexus-cert/scripts/cleanup-hook.sh mode change 100644 => 100755 templates/shared_services/nexus-cert/scripts/json-to-env.sh mode change 100644 => 100755 templates/shared_services/nexus-cert/scripts/letsencrypt.sh mode change 100644 => 100755 templates/shared_services/nexus-cert/scripts/outputs.sh diff --git a/templates/shared_services/nexus-cert/Dockerfile.tmpl b/templates/shared_services/nexus-cert/Dockerfile.tmpl old mode 100644 new mode 100755 diff --git a/templates/shared_services/nexus-cert/azure.json b/templates/shared_services/nexus-cert/azure.json old mode 100644 new mode 100755 diff --git a/templates/shared_services/nexus-cert/parameters.json b/templates/shared_services/nexus-cert/parameters.json old mode 100644 new mode 100755 diff --git a/templates/shared_services/nexus-cert/porter.yaml b/templates/shared_services/nexus-cert/porter.yaml old mode 100644 new mode 100755 diff --git a/templates/shared_services/nexus-cert/scripts/auth-hook.sh b/templates/shared_services/nexus-cert/scripts/auth-hook.sh old mode 100644 new mode 100755 diff --git a/templates/shared_services/nexus-cert/scripts/cleanup-hook.sh b/templates/shared_services/nexus-cert/scripts/cleanup-hook.sh old mode 100644 new mode 100755 diff --git a/templates/shared_services/nexus-cert/scripts/json-to-env.sh b/templates/shared_services/nexus-cert/scripts/json-to-env.sh old mode 100644 new mode 100755 diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh old mode 100644 new mode 100755 diff --git a/templates/shared_services/nexus-cert/scripts/outputs.sh b/templates/shared_services/nexus-cert/scripts/outputs.sh old mode 100644 new mode 100755 From 4635bb7243efae35800bffb92abf78be83bbd16c Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 25 Mar 2022 11:05:02 +0000 Subject: [PATCH 039/142] Get cert in tf --- .../sonatype-nexus/terraform/cloud-config.yaml | 14 ++++++++++++-- .../sonatype-nexus/terraform/data.tf | 5 +++++ .../shared_services/sonatype-nexus/terraform/vm.tf | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index c946be04c3..69621d3e0f 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -17,6 +17,7 @@ packages: - docker-compose - gnupg2 - pass + - default-jre # create the docker group groups: @@ -39,6 +40,12 @@ write_files: - path: /etc/nexus-data/nexus.properties content: | nexus.skipDefaultRepositories=true + application-port-ssl=8443 + permissions: '0755' + # create the ssl cert + - path: /home/adminuser/ssl.pem + content: | + ${ssl_certificate_base64} permissions: '0755' # Write a script that will reset the admin password for Nexus to the one TF generated - path: /home/adminuser/reset-nexus-password.sh @@ -87,9 +94,12 @@ runcmd: - export DEBIAN_FRONTEND=noninteractive # Give the Nexus process write permissions on the folder mounted as persistent volume - chown -R 200 /etc/nexus-data - # Run the nexus container (exposing port 8081 and mapping volume for nexus config) + # Prepare ssl certificate + - openssl pkcs12 -export -out ssl.pkcs12 -in /home/adminuser/ssl.pem + - keytool -v -importkeystore -srckeystore ssl.pkcs12 -srcstoretype PKCS12 -destkeystore /etc/nexus-data/etc/ssl/keystore.jks -deststoretype JKS + # Run the nexus container (mapping volume for nexus config) - docker run -d - -p 80:8081 -p 443:8081 + -p 80:8081 -p 443:8443 -v /etc/nexus-data:/nexus-data --restart always --name nexus diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus/terraform/data.tf index 8cdd03dba5..b4a2979910 100644 --- a/templates/shared_services/sonatype-nexus/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus/terraform/data.tf @@ -24,6 +24,11 @@ data "azurerm_key_vault" "kv" { resource_group_name = local.core_resource_group_name } +data "azurerm_key_vault_certificate" "ssl_certificate" { + name = "nexus-letsencrypt" + key_vault_id = data.azurerm_key_vault.kv.id +} + data "azurerm_storage_account" "nexus" { name = local.storage_account_name resource_group_name = local.core_resource_group_name diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 49a7a4a8e0..71a8b97df8 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -120,5 +120,6 @@ data "template_file" "nexus_config" { template = file("${path.module}/cloud-config.yaml") vars = { nexus_admin_password = random_password.nexus_admin_password.result + ssl_certificate_base64 = data.azurerm_key_vault_certificate.ssl_certificate.certificate_data_base64 } } From 6613e2111faf8b45a11b5ecb84669f37b98d6085 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 28 Mar 2022 13:22:59 +0000 Subject: [PATCH 040/142] Added az cli get cert --- .../terraform/cloud-config.yaml | 16 ++++++------ .../sonatype-nexus/terraform/data.tf | 5 ---- .../sonatype-nexus/terraform/vm.tf | 25 +++++++++++++++++-- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 69621d3e0f..479a799abd 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -1,7 +1,6 @@ --- #cloud-config package_upgrade: true -package_update: true apt: sources: docker.list: @@ -9,6 +8,11 @@ apt: https://download.docker.com/linux/ubuntu $RELEASE stable keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 keyserver: hkp://keyserver.ubuntu.com:80 + azure-cli.list: + source: deb [arch=amd64] + https://packages.microsoft.com/repos/azure-cli/ $RELEASE main + keyid: BC528686B50D79E339D3721CEB3E94ADBE1229CF + keyserver: hkp://keyserver.ubuntu.com:80 packages: - docker-ce @@ -17,6 +21,7 @@ packages: - docker-compose - gnupg2 - pass + - azure-cli - default-jre # create the docker group @@ -42,11 +47,6 @@ write_files: nexus.skipDefaultRepositories=true application-port-ssl=8443 permissions: '0755' - # create the ssl cert - - path: /home/adminuser/ssl.pem - content: | - ${ssl_certificate_base64} - permissions: '0755' # Write a script that will reset the admin password for Nexus to the one TF generated - path: /home/adminuser/reset-nexus-password.sh content: | @@ -95,7 +95,9 @@ runcmd: # Give the Nexus process write permissions on the folder mounted as persistent volume - chown -R 200 /etc/nexus-data # Prepare ssl certificate - - openssl pkcs12 -export -out ssl.pkcs12 -in /home/adminuser/ssl.pem + - az login --identity + - az keyvault certificate download --vault-name ${vault_name} -n {ssl_cert_name} -f cert.pem + - openssl pkcs12 -export -out ssl.pkcs12 -in cert.pem - keytool -v -importkeystore -srckeystore ssl.pkcs12 -srcstoretype PKCS12 -destkeystore /etc/nexus-data/etc/ssl/keystore.jks -deststoretype JKS # Run the nexus container (mapping volume for nexus config) - docker run -d diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus/terraform/data.tf index b4a2979910..8cdd03dba5 100644 --- a/templates/shared_services/sonatype-nexus/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus/terraform/data.tf @@ -24,11 +24,6 @@ data "azurerm_key_vault" "kv" { resource_group_name = local.core_resource_group_name } -data "azurerm_key_vault_certificate" "ssl_certificate" { - name = "nexus-letsencrypt" - key_vault_id = data.azurerm_key_vault.kv.id -} - data "azurerm_storage_account" "nexus" { name = local.storage_account_name resource_group_name = local.core_resource_group_name diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 71a8b97df8..f5004160eb 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -70,6 +70,21 @@ resource "azurerm_key_vault_secret" "nexus_admin_password" { key_vault_id = data.azurerm_key_vault.kv.id } +resource "azurerm_user_assigned_identity" "nexus_msi" { + name = "id-nexus-${var.tre_id}" + location = var.location + resource_group_name = local.core_resource_group_name + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_key_vault_access_policy" "nexus_msi" { + key_vault_id = data.azurerm_key_vault.kv.id + tenant_id = azurerm_user_assigned_identity.nexus_msi.tenant_id + object_id = azurerm_user_assigned_identity.nexus_msi.principal_id + + certificate_permissions = ["Get"] +} + resource "azurerm_linux_virtual_machine" "nexus" { name = "nexus-${var.tre_id}" resource_group_name = local.core_resource_group_name @@ -98,12 +113,17 @@ resource "azurerm_linux_virtual_machine" "nexus" { } identity { - type = "SystemAssigned" + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.nexus_msi.id] } boot_diagnostics { storage_account_uri = data.azurerm_storage_account.nexus.primary_blob_endpoint } + + depends_on = [ + azurerm_key_vault_access_policy.nexus_msi + ] } data "template_cloudinit_config" "nexus_config" { @@ -120,6 +140,7 @@ data "template_file" "nexus_config" { template = file("${path.module}/cloud-config.yaml") vars = { nexus_admin_password = random_password.nexus_admin_password.result - ssl_certificate_base64 = data.azurerm_key_vault_certificate.ssl_certificate.certificate_data_base64 + vault_name = data.azurerm_key_vault.kv.name + ssl_cert_name = "nexus-letsencrypt" } } From d1d885fcccf5a116b13b0a4745b822df58228e5f Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 28 Mar 2022 13:25:44 +0000 Subject: [PATCH 041/142] Amended prune job --- .../sonatype-nexus/terraform/cloud-config.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 479a799abd..f8ea2d42ea 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -34,12 +34,18 @@ system_info: groups: [docker] write_files: - # a weekly cron job to have docker free disk space - - path: /etc/cron.weekly/docker-prune + - path: /etc/cron.hourly/docker-prune + # An hourly cron job to have docker free disk space. Running this frquently + # since disk might get full fast, but we prune only when free space is low. content: | #!/bin/bash set -o errexit - docker system prune -f + used_percent=$(df / --output=pcent | tail -1 | sed 's/[^0-9]//g') + echo "Used disk space percent: $${used_percent}" + if (( used_percent > 60 )); then + echo "Free space too low, pruning..." + docker system prune -f + fi permissions: '0755' # ensure Nexus doesn't create default repositories - path: /etc/nexus-data/nexus.properties From 6c6382680c9a34eff80fe9bddb48f675d4f5caee Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 28 Mar 2022 13:27:22 +0000 Subject: [PATCH 042/142] Added msi id to login --- .../shared_services/sonatype-nexus/terraform/cloud-config.yaml | 2 +- templates/shared_services/sonatype-nexus/terraform/vm.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index f8ea2d42ea..a027d7b11f 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -101,7 +101,7 @@ runcmd: # Give the Nexus process write permissions on the folder mounted as persistent volume - chown -R 200 /etc/nexus-data # Prepare ssl certificate - - az login --identity + - az login --identity -u ${msi_id} - az keyvault certificate download --vault-name ${vault_name} -n {ssl_cert_name} -f cert.pem - openssl pkcs12 -export -out ssl.pkcs12 -in cert.pem - keytool -v -importkeystore -srckeystore ssl.pkcs12 -srcstoretype PKCS12 -destkeystore /etc/nexus-data/etc/ssl/keystore.jks -deststoretype JKS diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index f5004160eb..058f144689 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -140,6 +140,7 @@ data "template_file" "nexus_config" { template = file("${path.module}/cloud-config.yaml") vars = { nexus_admin_password = random_password.nexus_admin_password.result + msi_id = azurerm_user_assigned_identity.nexus_msi.id vault_name = data.azurerm_key_vault.kv.name ssl_cert_name = "nexus-letsencrypt" } From d9410b67dd53c4664365afb0bc51f2db298312fb Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 29 Mar 2022 10:41:22 +0000 Subject: [PATCH 043/142] Amended msi and exported cert pwd --- .../nexus-cert/scripts/letsencrypt.sh | 5 +++++ .../sonatype-nexus/terraform/cloud-config.yaml | 5 ++--- .../sonatype-nexus/terraform/variables.tf | 2 +- .../shared_services/sonatype-nexus/terraform/vm.tf | 12 ++++++++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh index eb4e3c4777..48e50432ab 100755 --- a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh +++ b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh @@ -77,6 +77,11 @@ if [[ -n ${KEYVAULT} ]]; then --password "${CERT_PASSWORD}" \ | jq -r '.sid') + # Save the certificate password to KV + az keyvault secret set --name nexus-letsencrypt-cert-password \ + --vault-name ${KEYVAULT} \ + --value "${CERT_PASSWORD}" + az network application-gateway ssl-cert update \ --resource-group "${RESOURCE_GROUP_NAME}" \ --gateway-name "${APPLICATION_GATEWAY}" \ diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index a027d7b11f..3b80378806 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -102,9 +102,8 @@ runcmd: - chown -R 200 /etc/nexus-data # Prepare ssl certificate - az login --identity -u ${msi_id} - - az keyvault certificate download --vault-name ${vault_name} -n {ssl_cert_name} -f cert.pem - - openssl pkcs12 -export -out ssl.pkcs12 -in cert.pem - - keytool -v -importkeystore -srckeystore ssl.pkcs12 -srcstoretype PKCS12 -destkeystore /etc/nexus-data/etc/ssl/keystore.jks -deststoretype JKS + - az keyvault secret download --vault-name ${vault_name} --name {ssl_cert_name} --file ssl.pfx + - keytool -v -importkeystore -srckeystore ssl.pfx -srcstoretype PKCS12 -destkeystore /etc/nexus-data/etc/ssl/keystore.jks -deststoretype JKS # Run the nexus container (mapping volume for nexus config) - docker run -d -p 80:8081 -p 443:8443 diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus/terraform/variables.tf index c5af1fe95c..636050a49d 100644 --- a/templates/shared_services/sonatype-nexus/terraform/variables.tf +++ b/templates/shared_services/sonatype-nexus/terraform/variables.tf @@ -11,5 +11,5 @@ variable "location" { variable "nexus_allowed_fqdns" { type = string description = "comma seperated string of allowed FQDNs for Nexus" - default = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com,repo.anaconda.com,*.docker.com,*.docker.io,conda.anaconda.org" + default = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com,keyserver.ubuntu.com,repo.anaconda.com,*.docker.com,*.docker.io,conda.anaconda.org,azure.archive.ubuntu.com, packages.microsoft.com" } diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 058f144689..2ab0363d7e 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -77,12 +77,18 @@ resource "azurerm_user_assigned_identity" "nexus_msi" { lifecycle { ignore_changes = [tags] } } +resource "azurerm_role_assignment" "kv_reader" { + scope = data.azurerm_key_vault.kv.id + role_definition_name = "Key Vault Reader" + principal_id = azurerm_user_assigned_identity.nexus_msi.principal_id +} + resource "azurerm_key_vault_access_policy" "nexus_msi" { key_vault_id = data.azurerm_key_vault.kv.id tenant_id = azurerm_user_assigned_identity.nexus_msi.tenant_id object_id = azurerm_user_assigned_identity.nexus_msi.principal_id - certificate_permissions = ["Get"] + secret_permissions = ["Get"] } resource "azurerm_linux_virtual_machine" "nexus" { @@ -122,7 +128,9 @@ resource "azurerm_linux_virtual_machine" "nexus" { } depends_on = [ - azurerm_key_vault_access_policy.nexus_msi + azurerm_role_assignment.kv_reader, + azurerm_key_vault_access_policy.nexus_msi, + azurerm_firewall_application_rule_collection.shared_subnet_nexus ] } From bed4a6ec6d3ea1178690ff1214cdde693a937f06 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 29 Mar 2022 22:07:26 +0000 Subject: [PATCH 044/142] Jetty configuration --- .../terraform/cloud-config.yaml | 28 ++++++++++++++++--- .../sonatype-nexus/terraform/data.tf | 10 +++++++ .../sonatype-nexus/terraform/vm.tf | 3 +- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 3b80378806..22e2351207 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -23,6 +23,7 @@ packages: - pass - azure-cli - default-jre + - xmlstarlet # create the docker group groups: @@ -47,11 +48,12 @@ write_files: docker system prune -f fi permissions: '0755' - # ensure Nexus doesn't create default repositories - - path: /etc/nexus-data/nexus.properties + # configure Nexus to disable default repos and set up SSL + - path: /etc/nexus-data/etc/nexus.properties content: | nexus.skipDefaultRepositories=true application-port-ssl=8443 + nexus-args=${jetty.etc}/jetty.xml,${jetty.etc}/jetty-http.xml,${jetty.etc}/jetty-requestlog.xml,/nexus-data/etc/jetty/jetty-https.xml permissions: '0755' # Write a script that will reset the admin password for Nexus to the one TF generated - path: /home/adminuser/reset-nexus-password.sh @@ -102,8 +104,14 @@ runcmd: - chown -R 200 /etc/nexus-data # Prepare ssl certificate - az login --identity -u ${msi_id} - - az keyvault secret download --vault-name ${vault_name} --name {ssl_cert_name} --file ssl.pfx - - keytool -v -importkeystore -srckeystore ssl.pfx -srcstoretype PKCS12 -destkeystore /etc/nexus-data/etc/ssl/keystore.jks -deststoretype JKS + # -- get cert from kv as secret so it contains private key + - az keyvault secret download --vault-name ${vault_name} --name ${ssl_cert_name} --file temp.pfx --encoding base64 + # -- az cli strips out password from cert which we need to re-add for jks by converting to PEM then back to PFX with pwd + - openssl pkcs12 -in temp.pfx -out temp.pem -nodes -password 'pass:' + - openssl pkcs12 -export -out nexus-ssl.pfx -in temp.pem -password "pass:${ssl_cert_password}" + # -- import into jks within nexus volume + - keytool -v -importkeystore -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 -destkeystore /etc/nexus-data/keystores/keystore.jks + -deststoretype JKS -srcstorepass "${ssl_cert_password}" -deststorepass "${ssl_cert_password}" # Run the nexus container (mapping volume for nexus config) - docker run -d -p 80:8081 -p 443:8443 @@ -112,5 +120,17 @@ runcmd: --name nexus --log-driver local sonatype/nexus3 + # Configure Jetty instance within Nexus to consume ssl cert + - mkdir /etc/nexus-data/etc/jetty + # -- we first need to copy the default Jetty config to the persistent volume so it isn't overwritten on restart + - docker exec -u root nexus cp /opt/sonatype/nexus/etc/jetty/jetty-https.xml /nexus-data/etc/jetty/ + # -- then we replace password values with the ssl cert keystore password + - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePassword']" -v "${ssl_cert_password}" jetty-https.xml + - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyManagerPassword']" -v "${ssl_cert_password}" jetty-https.xml + - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePassword']" -v "${ssl_cert_password}" jetty-https.xml + # -- update the location of our keystore + - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePath']" -v /nexus-data/keystores/keystore.jks jetty-https.xml + - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePath']" -v /nexus-data/keystores/keystore.jks jetty-https.xml + - docker restart nexus # Run the script to reset Nexus password (we write this as a script file first to avoid tricky handling of special chars in yml) - bash /home/adminuser/reset-nexus-password.sh diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus/terraform/data.tf index 8cdd03dba5..7728f4cd9b 100644 --- a/templates/shared_services/sonatype-nexus/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus/terraform/data.tf @@ -24,6 +24,16 @@ data "azurerm_key_vault" "kv" { resource_group_name = local.core_resource_group_name } +data "azurerm_key_vault_certificate" "nexus_cert" { + name = "nexus-letsencrypt" + key_vault_id = data.azurerm_key_vault.kv.id +} + +data "azurerm_key_vault_secret" "nexus_cert_password" { + name = "nexus-letsencrypt-cert-password" + key_vault_id = data.azurerm_key_vault.kv.id +} + data "azurerm_storage_account" "nexus" { name = local.storage_account_name resource_group_name = local.core_resource_group_name diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 2ab0363d7e..4d551c1527 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -150,6 +150,7 @@ data "template_file" "nexus_config" { nexus_admin_password = random_password.nexus_admin_password.result msi_id = azurerm_user_assigned_identity.nexus_msi.id vault_name = data.azurerm_key_vault.kv.name - ssl_cert_name = "nexus-letsencrypt" + ssl_cert_name = data.azurerm_key_vault_certificate.nexus_cert.name + ssl_cert_password = data.azurerm_key_vault_secret.nexus_cert_password.value } } From b4de6ab1d755ed46a289756ee038e061b5e6ce35 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 29 Mar 2022 23:14:03 +0000 Subject: [PATCH 045/142] Escape jetty vars --- .../shared_services/sonatype-nexus/terraform/cloud-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 22e2351207..1c78612d93 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -53,7 +53,7 @@ write_files: content: | nexus.skipDefaultRepositories=true application-port-ssl=8443 - nexus-args=${jetty.etc}/jetty.xml,${jetty.etc}/jetty-http.xml,${jetty.etc}/jetty-requestlog.xml,/nexus-data/etc/jetty/jetty-https.xml + nexus-args=$${jetty.etc}/jetty.xml,$${jetty.etc}/jetty-http.xml,$${jetty.etc}/jetty-requestlog.xml,/nexus-data/etc/jetty/jetty-https.xml permissions: '0755' # Write a script that will reset the admin password for Nexus to the one TF generated - path: /home/adminuser/reset-nexus-password.sh From 4f334365ddfc5ab3be70c69e4b6c8d2b02cc76e6 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 4 Apr 2022 23:40:57 +0000 Subject: [PATCH 046/142] Password script fixes --- .../terraform/cloud-config.yaml | 100 ++++++++++++------ .../sonatype-nexus/terraform/vm.tf | 2 +- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 1c78612d93..136526b957 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -48,20 +48,78 @@ write_files: docker system prune -f fi permissions: '0755' - # configure Nexus to disable default repos and set up SSL + + # Configure Nexus to disable default repos - path: /etc/nexus-data/etc/nexus.properties content: | nexus.skipDefaultRepositories=true + permissions: '0755' + + # Set up Nexus to serve https using SSL cert + - path: /home/adminuser/configure-nexus-ssl.sh + content: | + # Prepare ssl certificate + az login --identity -u ${msi_id} + # -- get cert from kv as secret so it contains private key + echo 'Getting cert and cert password from Keyvault...' + az keyvault secret download --vault-name ${vault_name} --name ${ssl_cert_name} --file temp.pfx --encoding base64 + CERT_PASSWORD=$(az keyvault secret show --vault-name ${vault_name} --name ${ssl_cert_password_name} -o tsv --query value) + # -- az cli strips out password from cert which we need to re-add for jks by converting to PEM then back to PFX with pwd + openssl pkcs12 -in temp.pfx -out temp.pem -nodes -password pass: + openssl pkcs12 -export -out nexus-ssl.pfx -in temp.pem -password "pass:$CERT_PASSWORD" + + # Import ssl cert to keystore within Nexus volume + keystore_timeout=300 + echo 'Checking for nexus-data/keystores directory...' + while [ ! -d /etc/nexus-data/keystores ]; do + # Wait for /keystore dir to be created by container first + if [ $keystore_timeout == 0 ]; then + echo 'ERROR - Timeout while waiting for Nexus to create nexus-data/keystores' + exit 1 + fi + sleep 1 + ((keystore_timeout--)) + done + echo 'Directory found. Importing ssl cert into new keystore at nexus-data/keystores/keystore.jks...' + keytool -v -importkeystore -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 -destkeystore /etc/nexus-data/keystores/keystore.jks \ + -deststoretype JKS -srcstorepass "$CERT_PASSWORD" -deststorepass "$CERT_PASSWORD" + + # Configure Jetty instance within Nexus to consume ssl cert + echo 'Modifying Nexus Jetty configuration to enable ssl...' + mkdir -p /etc/nexus-data/etc/jetty + # -- we first need to copy the default Jetty config to the persistent volume so it isn't overwritten on restart + docker exec -u root nexus cp /opt/sonatype/nexus/etc/jetty/jetty-https.xml /nexus-data/etc/jetty/ + # -- then we replace password values with the ssl cert keystore password + xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePassword']" \ + -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml + xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyManagerPassword']" \ + -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml + xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePassword']" \ + -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml + # -- then update the location of our keystore + xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePath']" \ + -v /nexus-data/keystores/keystore.jks /etc/nexus-data/etc/jetty/jetty-https.xml + xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePath']" \ + -v /nexus-data/keystores/keystore.jks /etc/nexus-data/etc/jetty/jetty-https.xml + + # Add jetty configuration and ssl port to Nexus properties + cat >> /etc/nexus-data/etc/nexus.properties <<'EOF' application-port-ssl=8443 nexus-args=$${jetty.etc}/jetty.xml,$${jetty.etc}/jetty-http.xml,$${jetty.etc}/jetty-requestlog.xml,/nexus-data/etc/jetty/jetty-https.xml - permissions: '0755' + EOF + + # Restart the container for changes to take effect + docker restart nexus + echo 'Nexus ssl configuration completed.' + permissions: '0744' + # Write a script that will reset the admin password for Nexus to the one TF generated - path: /home/adminuser/reset-nexus-password.sh content: | #!/bin/bash # Get the current password so we can post to the API # (this is created in the /nexus-data mounted volume as part of Nexus container start-up in cloud-init) - password_timeout=120 + password_timeout=300 echo 'Checking for Nexus admin password file...' while [ ! -f /etc/nexus-data/admin.password ]; do # We must first wait for the file to be created @@ -75,18 +133,18 @@ write_files: CURRENT_PASSWORD=$(cat /etc/nexus-data/admin.password) # Set our own admin password so we can connect to the Nexus repository manager later on using TF KV secret - reset_timeout=120 - echo 'Nexus default admin password found. Resetting...' + reset_timeout=300 + echo "Nexus default admin password found ($CURRENT_PASSWORD). Resetting..." # While the container is starting up it may return a number of transient errors (i.e. connection reset) which we need to retry until operational # NOTE: we can't use curl's built-in retry flags as it doesn't catch for the connection reset response res=1 while test "$res" != "0"; do curl -ifu admin:$CURRENT_PASSWORD -XPUT -H 'Content-Type:text/plain' --data '${nexus_admin_password}' \ http://localhost/service/rest/v1/security/users/admin/change-password - $res=$? + res=$? echo "Attempt to reset password finished with code $res" if test "$res" == "0"; then - echo 'Password reset successfully. Admin can now log in with secret stored in KeyVault. Nexus configuration complete.' + echo 'Password reset successfully. Admin can now log in with secret stored in KeyVault.' else if [ $reset_timeout == 0 ]; then echo 'ERROR - Timeout while trying to reset Nexus admin password' @@ -100,19 +158,9 @@ write_files: runcmd: - export DEBIAN_FRONTEND=noninteractive - # Give the Nexus process write permissions on the folder mounted as persistent volume + # Give the Nexus process write permissions on the folder mounted as persistent volume - chown -R 200 /etc/nexus-data - # Prepare ssl certificate - - az login --identity -u ${msi_id} - # -- get cert from kv as secret so it contains private key - - az keyvault secret download --vault-name ${vault_name} --name ${ssl_cert_name} --file temp.pfx --encoding base64 - # -- az cli strips out password from cert which we need to re-add for jks by converting to PEM then back to PFX with pwd - - openssl pkcs12 -in temp.pfx -out temp.pem -nodes -password 'pass:' - - openssl pkcs12 -export -out nexus-ssl.pfx -in temp.pem -password "pass:${ssl_cert_password}" - # -- import into jks within nexus volume - - keytool -v -importkeystore -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 -destkeystore /etc/nexus-data/keystores/keystore.jks - -deststoretype JKS -srcstorepass "${ssl_cert_password}" -deststorepass "${ssl_cert_password}" - # Run the nexus container (mapping volume for nexus config) + # Run the nexus container with mapped volume for nexus config - docker run -d -p 80:8081 -p 443:8443 -v /etc/nexus-data:/nexus-data @@ -120,17 +168,5 @@ runcmd: --name nexus --log-driver local sonatype/nexus3 - # Configure Jetty instance within Nexus to consume ssl cert - - mkdir /etc/nexus-data/etc/jetty - # -- we first need to copy the default Jetty config to the persistent volume so it isn't overwritten on restart - - docker exec -u root nexus cp /opt/sonatype/nexus/etc/jetty/jetty-https.xml /nexus-data/etc/jetty/ - # -- then we replace password values with the ssl cert keystore password - - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePassword']" -v "${ssl_cert_password}" jetty-https.xml - - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyManagerPassword']" -v "${ssl_cert_password}" jetty-https.xml - - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePassword']" -v "${ssl_cert_password}" jetty-https.xml - # -- update the location of our keystore - - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePath']" -v /nexus-data/keystores/keystore.jks jetty-https.xml - - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePath']" -v /nexus-data/keystores/keystore.jks jetty-https.xml - - docker restart nexus - # Run the script to reset Nexus password (we write this as a script file first to avoid tricky handling of special chars in yml) - bash /home/adminuser/reset-nexus-password.sh + - bash /home/adminuser/configure-nexus-ssl.sh diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 4d551c1527..8c640aa480 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -151,6 +151,6 @@ data "template_file" "nexus_config" { msi_id = azurerm_user_assigned_identity.nexus_msi.id vault_name = data.azurerm_key_vault.kv.name ssl_cert_name = data.azurerm_key_vault_certificate.nexus_cert.name - ssl_cert_password = data.azurerm_key_vault_secret.nexus_cert_password.value + ssl_cert_password_name = data.azurerm_key_vault_secret.nexus_cert_password.name } } From f7ac643d3acee1638a1db677b6deeef5a7ef061f Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 4 Apr 2022 23:55:00 +0000 Subject: [PATCH 047/142] Amended networking to use module --- templates/workspaces/base/terraform/network/data.tf | 5 +++++ .../workspaces/base/terraform/network/zone_links.tf | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/templates/workspaces/base/terraform/network/data.tf b/templates/workspaces/base/terraform/network/data.tf index 929ca9d36f..8f293553a2 100644 --- a/templates/workspaces/base/terraform/network/data.tf +++ b/templates/workspaces/base/terraform/network/data.tf @@ -81,3 +81,8 @@ data "azurerm_private_dns_zone" "postgres" { name = "privatelink.postgres.database.azure.com" resource_group_name = local.core_resource_group_name } + +data "azurerm_private_dns_zone" "nexus" { + name = "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" + resource_group_name = local.core_resource_group_name +} diff --git a/templates/workspaces/base/terraform/network/zone_links.tf b/templates/workspaces/base/terraform/network/zone_links.tf index 84947ad82e..7d3da49b31 100644 --- a/templates/workspaces/base/terraform/network/zone_links.tf +++ b/templates/workspaces/base/terraform/network/zone_links.tf @@ -88,3 +88,12 @@ resource "azurerm_private_dns_zone_virtual_network_link" "postgreslink" { lifecycle { ignore_changes = [tags] } } + +resource "azurerm_private_dns_zone_virtual_network_link" "nexuslink" { + name = "nexuslink-${local.workspace_resource_name_suffix}" + resource_group_name = local.core_resource_group_name + private_dns_zone_name = data.azurerm_private_dns_zone.nexus.name + virtual_network_id = azurerm_virtual_network.ws.id + + lifecycle { ignore_changes = [tags] } +} From d5612c21cc58166e027efd46d05ce83440703b94 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 5 Apr 2022 00:00:03 +0000 Subject: [PATCH 048/142] Use https in config script --- .../shared_services/sonatype-nexus/scripts/configure_nexus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index d52556fc29..69a84a973b 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -38,7 +38,7 @@ while [ "$1" != "" ]; do shift # remove the current value for `$1` and use the next done -export NEXUS_URL="http://nexus-${tre_id}.${location}.cloudapp.azure.com" # TODO: change to https once ssl cert is added +export NEXUS_URL="https://nexus-${tre_id}.${location}.cloudapp.azure.com" export NEXUS_ADMIN_PASSWORD_NAME="nexus-admin-password" export KEYVAULT_NAME="kv-${tre_id}" export NEXUS_PASS=$(az keyvault secret show --name ${NEXUS_ADMIN_PASSWORD_NAME} --vault-name ${KEYVAULT_NAME} -o json | jq -r '.value') From 300527e0222abbbb61fcaf905f02ffd16c52731d Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 5 Apr 2022 13:04:41 +0000 Subject: [PATCH 049/142] Removed res proc location variable --- resource_processor/shared/config.py | 1 - templates/core/terraform/json-to-env.sh | 4 ---- templates/core/terraform/outputs.tf | 4 ---- .../user_resources/guacamole-azure-linuxvm/porter.yaml | 4 ---- .../guacamole-azure-linuxvm/terraform/locals.tf | 8 ++++++-- .../user_resources/guacamole-azure-windowsvm/porter.yaml | 4 ---- .../guacamole-azure-windowsvm/terraform/locals.tf | 8 ++++++-- 7 files changed, 12 insertions(+), 21 deletions(-) diff --git a/resource_processor/shared/config.py b/resource_processor/shared/config.py index b36eeca0ca..63f5fa6dcb 100644 --- a/resource_processor/shared/config.py +++ b/resource_processor/shared/config.py @@ -11,7 +11,6 @@ def get_config(logger_adapter) -> dict: config["deployment_status_queue"] = os.environ["SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE"] config["resource_request_queue"] = os.environ["SERVICE_BUS_RESOURCE_REQUEST_QUEUE"] config["service_bus_namespace"] = os.environ["SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE"] - config["location"] = os.environ["LOCATION"] config["vmss_msi_id"] = os.environ.get("VMSS_MSI_ID", None) config["number_processes"] = os.environ.get("NUMBER_PROCESSES", "1") diff --git a/templates/core/terraform/json-to-env.sh b/templates/core/terraform/json-to-env.sh index d88c89ca95..57b797b5a9 100755 --- a/templates/core/terraform/json-to-env.sh +++ b/templates/core/terraform/json-to-env.sh @@ -76,10 +76,6 @@ jq -r ' { "path": "registry_server", "env_var": "REGISTRY_SERVER" - }, - { - "path": "location", - "env_var": "LOCATION" } ] as $env_vars_to_extract diff --git a/templates/core/terraform/outputs.tf b/templates/core/terraform/outputs.tf index 6ccfa30b10..00ff260525 100644 --- a/templates/core/terraform/outputs.tf +++ b/templates/core/terraform/outputs.tf @@ -76,7 +76,3 @@ output "terraform_state_container_name" { output "registry_server" { value = var.docker_registry_server } - -output "location" { - value = var.location -} diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index b1521c9506..3a7b39621b 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -30,9 +30,6 @@ parameters: type: string description: "An Id for this installation" env: id - - name: location - type: string - description: "The location the TRE is deployed in" - name: tfstate_resource_group_name type: string description: "Resource group containing the Terraform state storage account" @@ -91,7 +88,6 @@ install: vars: workspace_id: "{{ bundle.parameters.workspace_id }}" tre_id: "{{ bundle.parameters.tre_id }}" - location: "{{ bundle.parameters.location }}" parent_service_id: "{{ bundle.parameters.parent_service_id }}" arm_client_id: "{{ bundle.credentials.azure_client_id }}" arm_client_secret: "{{ bundle.credentials.azure_client_secret }}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf index 5b191804bd..97ab00acfe 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf @@ -1,3 +1,7 @@ +data "azurerm_resource_group" "core" { + name = "rg-${var.tre_id}" +} + locals { short_service_id = substr(var.tre_resource_id, -4, -1) short_workspace_id = substr(var.workspace_id, -4, -1) @@ -5,11 +9,11 @@ locals { workspace_resource_name_suffix = "${var.tre_id}-ws-${local.short_workspace_id}" service_resource_name_suffix = "${var.tre_id}-ws-${local.short_workspace_id}-svc-${local.short_service_id}" core_vnet = "vnet-${var.tre_id}" - core_resource_group_name = "rg-${var.tre_id}" + core_resource_group_name = data.azurerm_resource_group.core.name vm_name = "linuxvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") storage_name = lower(replace("stg${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", "")) - nexus_proxy_url = "https://nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" + nexus_proxy_url = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" vm_size = { "2 CPU | 8GB RAM" = { value = "Standard_D2s_v5" }, "4 CPU | 16GB RAM" = { value = "Standard_D4s_v5" }, diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index 15304736f2..9b672271ce 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -30,9 +30,6 @@ parameters: type: string description: "An Id for this installation" env: id - - name: location - type: string - description: "The location the TRE is deployed in" - name: tfstate_resource_group_name type: string description: "Resource group containing the Terraform state storage account" @@ -91,7 +88,6 @@ install: vars: workspace_id: "{{ bundle.parameters.workspace_id }}" tre_id: "{{ bundle.parameters.tre_id }}" - location: "{{ bundle.parameters.location }}" parent_service_id: "{{ bundle.parameters.parent_service_id }}" arm_client_id: "{{ bundle.credentials.azure_client_id }}" arm_client_secret: "{{ bundle.credentials.azure_client_secret }}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf index 009165b36d..d3fff01a8b 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf @@ -1,3 +1,7 @@ +data "azurerm_resource_group" "core" { + name = "rg-${var.tre_id}" +} + locals { short_service_id = substr(var.tre_resource_id, -4, -1) short_workspace_id = substr(var.workspace_id, -4, -1) @@ -5,11 +9,11 @@ locals { workspace_resource_name_suffix = "${var.tre_id}-ws-${local.short_workspace_id}" service_resource_name_suffix = "${var.tre_id}-ws-${local.short_workspace_id}-svc-${local.short_service_id}" core_vnet = "vnet-${var.tre_id}" - core_resource_group_name = "rg-${var.tre_id}" + core_resource_group_name = data.azurerm_resource_group.core.name vm_name = "windowsvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") storage_name = lower(replace("stg${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", "")) - nexus_proxy_url = "https://nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" + nexus_proxy_url = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" vm_size = { "2 CPU | 8GB RAM" = { value = "Standard_D2s_v5" }, "4 CPU | 16GB RAM" = { value = "Standard_D4s_v5" }, From 50bd946b5072295d06376b871bd8dbcb8fb68250 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 5 Apr 2022 22:30:51 +0000 Subject: [PATCH 050/142] Potential linting fix --- .../sonatype-nexus/terraform/cloud-config.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 136526b957..4cf6ebf136 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -161,12 +161,10 @@ runcmd: # Give the Nexus process write permissions on the folder mounted as persistent volume - chown -R 200 /etc/nexus-data # Run the nexus container with mapped volume for nexus config - - docker run -d - -p 80:8081 -p 443:8443 - -v /etc/nexus-data:/nexus-data - --restart always - --name nexus - --log-driver local - sonatype/nexus3 + - docker run -d -p 80:8081 -p 443:8443 -v /etc/nexus-data:/nexus-data + --restart always + --name nexus + --log-driver local + sonatype/nexus3 - bash /home/adminuser/reset-nexus-password.sh - bash /home/adminuser/configure-nexus-ssl.sh From bef085a499d0ade046167304390c43efe223fde3 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 6 Apr 2022 11:15:50 +0000 Subject: [PATCH 051/142] Linting fixes --- .../nexus-cert/scripts/auth-hook.sh | 1 + .../nexus-cert/scripts/letsencrypt.sh | 20 +++++----- .../nexus-cert/scripts/outputs.sh | 6 +-- .../sonatype-nexus/scripts/configure_nexus.sh | 22 +++++------ .../terraform/cloud-config.yaml | 37 +++++++++++-------- 5 files changed, 48 insertions(+), 38 deletions(-) diff --git a/templates/shared_services/nexus-cert/scripts/auth-hook.sh b/templates/shared_services/nexus-cert/scripts/auth-hook.sh index ede440f231..1963ea262a 100755 --- a/templates/shared_services/nexus-cert/scripts/auth-hook.sh +++ b/templates/shared_services/nexus-cert/scripts/auth-hook.sh @@ -7,6 +7,7 @@ EOF az storage blob upload \ --account-name "${STORAGE_ACCOUNT}" \ --auth-mode login \ + # shellcheck disable=SC2016 --container-name '$web' \ --file 'validation.txt' \ --name ".well-known/acme-challenge/${CERTBOT_TOKEN}" \ diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh index 48e50432ab..6c1c7d7e23 100755 --- a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh +++ b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -script_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") if [[ -z ${STORAGE_ACCOUNT} ]]; then echo "STORAGE_ACCOUNT not set" exit 1 @@ -18,6 +18,7 @@ EOF indexExists=$(az storage blob list -o json \ --account-name "${STORAGE_ACCOUNT}" \ --auth-mode login \ + # shellcheck disable=SC2016 --container-name '$web' \ --query "[?name=='index.html'].name" \ | jq 'length') @@ -28,6 +29,7 @@ if [[ ${indexExists} -lt 1 ]]; then az storage blob upload \ --account-name "${STORAGE_ACCOUNT}" \ --auth-mode login \ + # shellcheck disable=SC2016 --container-name '$web' \ --file index.html \ --name index.html \ @@ -46,14 +48,14 @@ mkdir -p "${ledir}/logs" # Initiate the ACME challange /opt/certbot/bin/certbot certonly \ - --config-dir ${ledir} \ - --work-dir ${ledir} \ - --logs-dir ${ledir}/logs \ + --config-dir "${ledir}" \ + --work-dir "${ledir}" \ + --logs-dir "${ledir}"/logs \ --manual \ --preferred-challenges=http \ - --manual-auth-hook ${script_dir}/auth-hook.sh \ - --manual-cleanup-hook ${script_dir}/cleanup-hook.sh \ - --domain ${FQDN} \ + --manual-auth-hook "${script_dir}"/auth-hook.sh \ + --manual-cleanup-hook "${script_dir}"/cleanup-hook.sh \ + --domain "${FQDN}" \ --non-interactive \ --agree-tos \ --register-unsafely-without-email @@ -71,7 +73,7 @@ openssl pkcs12 -export \ if [[ -n ${KEYVAULT} ]]; then sid=$(az keyvault certificate import \ -o json \ - --vault-name ${KEYVAULT} \ + --vault-name "${KEYVAULT}" \ --name 'nexus-letsencrypt' \ --file "${CERT_DIR}/aci.pfx" \ --password "${CERT_PASSWORD}" \ @@ -79,7 +81,7 @@ if [[ -n ${KEYVAULT} ]]; then # Save the certificate password to KV az keyvault secret set --name nexus-letsencrypt-cert-password \ - --vault-name ${KEYVAULT} \ + --vault-name "${KEYVAULT}" \ --value "${CERT_PASSWORD}" az network application-gateway ssl-cert update \ diff --git a/templates/shared_services/nexus-cert/scripts/outputs.sh b/templates/shared_services/nexus-cert/scripts/outputs.sh index 04d21c9452..11b11a4114 100755 --- a/templates/shared_services/nexus-cert/scripts/outputs.sh +++ b/templates/shared_services/nexus-cert/scripts/outputs.sh @@ -5,9 +5,9 @@ if [ ! -f ../tre_output.json ]; then # Connect to the remote backend of Terraform export TF_LOG="" terraform init -input=false -backend=true -reconfigure -upgrade \ - -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ - -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ - -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ + -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name:?}" \ + -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name:?}" \ + -backend-config="container_name=${TF_VAR_terraform_state_container_name:?}" \ -backend-config="key=${TRE_ID}" # Convert the output to json diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh index 69a84a973b..433af77c21 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -38,10 +38,10 @@ while [ "$1" != "" ]; do shift # remove the current value for `$1` and use the next done -export NEXUS_URL="https://nexus-${tre_id}.${location}.cloudapp.azure.com" -export NEXUS_ADMIN_PASSWORD_NAME="nexus-admin-password" -export KEYVAULT_NAME="kv-${tre_id}" -export NEXUS_PASS=$(az keyvault secret show --name ${NEXUS_ADMIN_PASSWORD_NAME} --vault-name ${KEYVAULT_NAME} -o json | jq -r '.value') +NEXUS_URL="https://nexus-${tre_id}.${location}.cloudapp.azure.com" +NEXUS_ADMIN_PASSWORD_NAME="nexus-admin-password" +KEYVAULT_NAME="kv-${tre_id}" +NEXUS_PASS=$(az keyvault secret show --name "${NEXUS_ADMIN_PASSWORD_NAME}" --vault-name "${KEYVAULT_NAME}" -o json | jq -r '.value') if [ -z "$NEXUS_PASS" ]; then echo "Unable to get the Nexus admin password from Keyvault. You may need to manually reset it in the Nexus host. Refer to the public Nexus documentation for more information." @@ -52,23 +52,23 @@ fi for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_config/*.json; do echo "Found config file: $filename. Sending to Nexus..." # Check if apt proxy - base_type=$( jq .baseType $filename | sed 's/"//g') - repo_type=$( jq .repoType $filename | sed 's/"//g') - repo_name=$(jq .name $filename | sed 's/"//g') + base_type=$( jq .baseType "$filename" | sed 's/"//g') + repo_type=$( jq .repoType "$filename" | sed 's/"//g') + repo_name=$(jq .name "$filename" | sed 's/"//g') base_url=$NEXUS_URL/service/rest/v1/repositories/$base_type/$repo_type full_url=$base_url/$repo_name - export STATUS_CODE=$(curl -iu admin:$NEXUS_PASS -X "GET" $full_url -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) + STATUS_CODE=$(curl -iu admin:"$NEXUS_PASS" -X "GET" "$full_url" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) echo "Response received from Nexus: $STATUS_CODE" if [[ ${STATUS_CODE} == 404 ]] then - curl -iu admin:$NEXUS_PASS -XPOST \ - $base_url \ + curl -iu admin:"$NEXUS_PASS" -XPOST \ + "$base_url" \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ - -d @$filename + -d @"$filename" else echo "$repo_type proxy for $repo_name already exists." fi diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 4cf6ebf136..1887165e6f 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -63,8 +63,9 @@ write_files: # -- get cert from kv as secret so it contains private key echo 'Getting cert and cert password from Keyvault...' az keyvault secret download --vault-name ${vault_name} --name ${ssl_cert_name} --file temp.pfx --encoding base64 - CERT_PASSWORD=$(az keyvault secret show --vault-name ${vault_name} --name ${ssl_cert_password_name} -o tsv --query value) - # -- az cli strips out password from cert which we need to re-add for jks by converting to PEM then back to PFX with pwd + CERT_PASSWORD=$(az keyvault secret show --vault-name ${vault_name} \ + --name ${ssl_cert_password_name} -o tsv --query value) + # -- az cli strips out password from cert so we re-add by converting to PEM then PFX with pwd openssl pkcs12 -in temp.pfx -out temp.pem -nodes -password pass: openssl pkcs12 -export -out nexus-ssl.pfx -in temp.pem -password "pass:$CERT_PASSWORD" @@ -80,26 +81,32 @@ write_files: sleep 1 ((keystore_timeout--)) done - echo 'Directory found. Importing ssl cert into new keystore at nexus-data/keystores/keystore.jks...' - keytool -v -importkeystore -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 -destkeystore /etc/nexus-data/keystores/keystore.jks \ + echo 'Directory found. Importing ssl cert into nexus-data/keystores/keystore.jks...' + keytool -v -importkeystore -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 \ + -destkeystore /etc/nexus-data/keystores/keystore.jks \ -deststoretype JKS -srcstorepass "$CERT_PASSWORD" -deststorepass "$CERT_PASSWORD" # Configure Jetty instance within Nexus to consume ssl cert echo 'Modifying Nexus Jetty configuration to enable ssl...' mkdir -p /etc/nexus-data/etc/jetty - # -- we first need to copy the default Jetty config to the persistent volume so it isn't overwritten on restart + # -- first need to copy default Jetty config to persistent volume so isn't overwritten on restart docker exec -u root nexus cp /opt/sonatype/nexus/etc/jetty/jetty-https.xml /nexus-data/etc/jetty/ # -- then we replace password values with the ssl cert keystore password - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePassword']" \ + xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePassword']" \ -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyManagerPassword']" \ + xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyManagerPassword']" \ -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePassword']" \ + xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePassword']" \ -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml # -- then update the location of our keystore - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePath']" \ + xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePath']" \ -v /nexus-data/keystores/keystore.jks /etc/nexus-data/etc/jetty/jetty-https.xml - xmlstarlet ed -P --inplace -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePath']" \ + xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePath']" \ -v /nexus-data/keystores/keystore.jks /etc/nexus-data/etc/jetty/jetty-https.xml # Add jetty configuration and ssl port to Nexus properties @@ -118,7 +125,7 @@ write_files: content: | #!/bin/bash # Get the current password so we can post to the API - # (this is created in the /nexus-data mounted volume as part of Nexus container start-up in cloud-init) + # (this is created in /nexus-data mounted volume as part of Nexus container start-up) password_timeout=300 echo 'Checking for Nexus admin password file...' while [ ! -f /etc/nexus-data/admin.password ]; do @@ -132,10 +139,10 @@ write_files: done CURRENT_PASSWORD=$(cat /etc/nexus-data/admin.password) - # Set our own admin password so we can connect to the Nexus repository manager later on using TF KV secret + # Set own admin password so we can connect to repository manager later on using TF KV secret reset_timeout=300 echo "Nexus default admin password found ($CURRENT_PASSWORD). Resetting..." - # While the container is starting up it may return a number of transient errors (i.e. connection reset) which we need to retry until operational + # While the container is starting up it may return a number of transient errors which we need to retry # NOTE: we can't use curl's built-in retry flags as it doesn't catch for the connection reset response res=1 while test "$res" != "0"; do @@ -158,9 +165,9 @@ write_files: runcmd: - export DEBIAN_FRONTEND=noninteractive - # Give the Nexus process write permissions on the folder mounted as persistent volume + # Give the Nexus process write permissions on the folder mounted as persistent volume - chown -R 200 /etc/nexus-data - # Run the nexus container with mapped volume for nexus config + # Run the nexus container with mapped volume for nexus config - docker run -d -p 80:8081 -p 443:8443 -v /etc/nexus-data:/nexus-data --restart always --name nexus From 78e6f914f65b702ea927142e64d16f215ca36d3e Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 6 Apr 2022 11:31:39 +0000 Subject: [PATCH 052/142] Linting directive positioning --- templates/shared_services/nexus-cert/scripts/auth-hook.sh | 2 +- templates/shared_services/nexus-cert/scripts/letsencrypt.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/shared_services/nexus-cert/scripts/auth-hook.sh b/templates/shared_services/nexus-cert/scripts/auth-hook.sh index 1963ea262a..ea87070267 100755 --- a/templates/shared_services/nexus-cert/scripts/auth-hook.sh +++ b/templates/shared_services/nexus-cert/scripts/auth-hook.sh @@ -4,10 +4,10 @@ cat << EOF > 'validation.txt' ${CERTBOT_VALIDATION} EOF +# shellcheck disable=SC2016 az storage blob upload \ --account-name "${STORAGE_ACCOUNT}" \ --auth-mode login \ - # shellcheck disable=SC2016 --container-name '$web' \ --file 'validation.txt' \ --name ".well-known/acme-challenge/${CERTBOT_TOKEN}" \ diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh index 6c1c7d7e23..0bf013a262 100755 --- a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh +++ b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh @@ -15,10 +15,10 @@ cat << EOF > index.html EOF +# shellcheck disable=SC2016 indexExists=$(az storage blob list -o json \ --account-name "${STORAGE_ACCOUNT}" \ --auth-mode login \ - # shellcheck disable=SC2016 --container-name '$web' \ --query "[?name=='index.html'].name" \ | jq 'length') @@ -26,10 +26,10 @@ indexExists=$(az storage blob list -o json \ if [[ ${indexExists} -lt 1 ]]; then echo "Uploading index.html file" + # shellcheck disable=SC2016 az storage blob upload \ --account-name "${STORAGE_ACCOUNT}" \ --auth-mode login \ - # shellcheck disable=SC2016 --container-name '$web' \ --file index.html \ --name index.html \ From 20c2ab3821c6f7afc772b78f4bfff5a088a4832d Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 6 Apr 2022 11:38:37 +0000 Subject: [PATCH 053/142] Gitea version bump --- templates/shared_services/gitea/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/gitea/version.txt b/templates/shared_services/gitea/version.txt index 1276d0254f..0a8da88258 100644 --- a/templates/shared_services/gitea/version.txt +++ b/templates/shared_services/gitea/version.txt @@ -1 +1 @@ -__version__ = "0.1.5" +__version__ = "0.1.6" From 4b147a2f2a67872bf0d884bb54403f4e2d2cf80f Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 6 Apr 2022 11:42:23 +0000 Subject: [PATCH 054/142] Terraform format --- templates/shared_services/sonatype-nexus/terraform/vm.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 8c640aa480..8ab94be297 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -147,10 +147,10 @@ data "template_cloudinit_config" "nexus_config" { data "template_file" "nexus_config" { template = file("${path.module}/cloud-config.yaml") vars = { - nexus_admin_password = random_password.nexus_admin_password.result - msi_id = azurerm_user_assigned_identity.nexus_msi.id - vault_name = data.azurerm_key_vault.kv.name - ssl_cert_name = data.azurerm_key_vault_certificate.nexus_cert.name + nexus_admin_password = random_password.nexus_admin_password.result + msi_id = azurerm_user_assigned_identity.nexus_msi.id + vault_name = data.azurerm_key_vault.kv.name + ssl_cert_name = data.azurerm_key_vault_certificate.nexus_cert.name ssl_cert_password_name = data.azurerm_key_vault_secret.nexus_cert_password.name } } From 7867b4557bb097bda4f03e0dcfb803a2435bf615 Mon Sep 17 00:00:00 2001 From: Stuart Leeks Date: Wed, 6 Apr 2022 14:35:24 +0100 Subject: [PATCH 055/142] Reorder linting to workaround superlinter bug with Terraform --- .../workflows/build_validation_develop.yml | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index bf4009b150..a3cfab2bc4 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -24,23 +24,6 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Lint code base - # the slim image is 2GB smaller and we don't use the extra stuff - uses: github/super-linter/slim@v4 - env: - VALIDATE_ALL_CODEBASE: false - DEFAULT_BRANCH: main - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VALIDATE_MARKDOWN: true - VALIDATE_PYTHON_FLAKE8: true - VALIDATE_YAML: true - VALIDATE_TERRAFORM_TFLINT: true - VALIDATE_JAVA: true - JAVA_FILE_NAME: checkstyle.xml - VALIDATE_BASH: true - VALIDATE_BASH_EXEC: true - VALIDATE_GITHUB_ACTIONS: true - - uses: dorny/paths-filter@v2 id: filter with: @@ -59,5 +42,23 @@ jobs: if: ${{ steps.filter.outputs.terraform == 'true' }} run: | find . -type d -name 'terraform' -not -path '*cnab*' -print0 \ - | xargs -0 -I{} sh -c 'echo "***** Validating: {} *****"; \ + | xargs -0 -I{} sh -c 'echo "***** Validating: {} *****"; \https://github.com/github/super-linter/issues/2433 terraform -chdir={} init -backend=false; terraform -chdir={} validate' + + - name: Lint code base + # the slim image is 2GB smaller and we don't use the extra stuff + # Moved this after the Terraform checks above due to this issue: https://github.com/github/super-linter/issues/2433 + uses: github/super-linter/slim@v4 + env: + VALIDATE_ALL_CODEBASE: false + DEFAULT_BRANCH: main + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_MARKDOWN: true + VALIDATE_PYTHON_FLAKE8: true + VALIDATE_YAML: true + VALIDATE_TERRAFORM_TFLINT: true + VALIDATE_JAVA: true + JAVA_FILE_NAME: checkstyle.xml + VALIDATE_BASH: true + VALIDATE_BASH_EXEC: true + VALIDATE_GITHUB_ACTIONS: true From a68eefe296a590d6ae42d247ba545da7232f423e Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 7 Apr 2022 11:13:03 +0000 Subject: [PATCH 056/142] Added nexus-cert to build and caching of letsencrypt --- .github/workflows/deploy_tre_reusable.yml | 6 ++++++ Makefile | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 0f8a8940ce..491f356c1b 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -349,6 +349,12 @@ jobs: TF_VAR_keyvault_purge_protection_enabled: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}" TF_VAR_stateful_resources_locked: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}" + # We need to cache letsencrypt local metadata on agent to avoid repeated challenges and blacklisting + - name: Cache Nexus Letsencrypt metadata + uses: actions/cache@v3 + with: + key: nexus letsencrypt + path: ./templates/shared_services/nexus-cert/letsencrypt - name: Install Nexus uses: ./.github/actions/devcontainer_run_command with: diff --git a/Makefile b/Makefile index dac4d4e6bc..9071149e6a 100644 --- a/Makefile +++ b/Makefile @@ -147,7 +147,7 @@ gitea-install: $(call target_title, "Installing Gitea") \ && make SHARED_SERVICE_KEY=shared-service-gitea terraform-shared-service-deploy DIR=./templates/shared_services/gitea/terraform -nexus-install: +nexus-install: nexus-cert-install nexus-letsencrypt $(call target_title, "Installing Nexus") \ && make SHARED_SERVICE_KEY=shared-service-sonatype-nexus terraform-shared-service-deploy DIR=./templates/shared_services/sonatype-nexus/terraform From dc1dc72fc2fd626da4b0d548dd9356475bf25e9b Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 7 Apr 2022 12:53:31 +0000 Subject: [PATCH 057/142] Adopted new shared service deploy method --- Makefile | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 5e92c7efb9..34f364e6e5 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ firewall-install: && $(MAKE) bundle-publish DIR=./templates/shared_services/firewall/ \ && $(MAKE) shared-service-register-and-deploy DIR=./templates/shared_services/firewall/ BUNDLE_TYPE=shared_service -nexus-install: +nexus-install: nexus-cert-install nexus-letsencrypt $(MAKE) bundle-build DIR=./templates/shared_services/sonatype-nexus/ \ && $(MAKE) bundle-publish DIR=./templates/shared_services/sonatype-nexus/ \ && $(MAKE) shared-service-register-and-deploy DIR=./templates/shared_services/sonatype-nexus/ BUNDLE_TYPE=shared_service @@ -99,6 +99,11 @@ gitea-install: && $(MAKE) bundle-publish DIR=./templates/shared_services/gitea/ \ && $(MAKE) shared-service-register-and-deploy DIR=./templates/shared_services/gitea/ BUNDLE_TYPE=shared_service +nexus-cert-install: + $(MAKE) bundle-build DIR=./templates/shared_services/nexus-cert/ \ + && $(MAKE) bundle-publish DIR=./templates/shared_services/nexus-cert/ \ + && $(MAKE) shared-service-register-and-deploy DIR=./templates/shared_services/nexus-cert/ BUNDLE_TYPE=shared_service + # A recipe for pushing images. Parameters: # 1. Image name suffix # 2. Version file path @@ -154,24 +159,8 @@ terraform-shared-service-deploy: && . ./devops/scripts/key_vault_list.sh \ && if [[ "$${TF_LOG}" == "DEBUG" ]]; then echo "TF DEBUG set - output supressed - see tflogs container for log file" && cd ${DIR} && ../../deploy_from_local.sh 1>/dev/null 2>/dev/null; else cd ${DIR} && ../../deploy_from_local.sh; fi; -firewall-install: - $(call target_title, "Installing Firewall") \ - && make SHARED_SERVICE_KEY=shared-service-firewall terraform-shared-service-deploy DIR=./templates/shared_services/firewall/terraform - -gitea-install: - $(call target_title, "Installing Gitea") \ - && make SHARED_SERVICE_KEY=shared-service-gitea terraform-shared-service-deploy DIR=./templates/shared_services/gitea/terraform - -nexus-install: nexus-cert-install nexus-letsencrypt - $(call target_title, "Installing Nexus") \ - && make SHARED_SERVICE_KEY=shared-service-sonatype-nexus terraform-shared-service-deploy DIR=./templates/shared_services/sonatype-nexus/terraform - # / End migration targets -nexus-cert-install: - $(call target_title, "Installing Nexus Cert") \ - && make SHARED_SERVICE_KEY=shared-service-nexus-cert terraform-shared-service-deploy DIR=./templates/shared_services/nexus-cert/terraform - nexus-letsencrypt: $(call target_title, "Requesting LetsEncrypt SSL certificate for Nexus") \ && . ./devops/scripts/check_dependencies.sh nodocker,certbot \ From 96deffb821d1ec2a6bb0e264d71e2a40e633efbd Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 7 Apr 2022 13:08:59 +0000 Subject: [PATCH 058/142] Added cron job to renew nexus cert --- .../sonatype-nexus/terraform/cloud-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 1887165e6f..aaa1402617 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -49,6 +49,15 @@ write_files: fi permissions: '0755' + - path: /etc/cron.daily/renew-nexus-cert + # Daily cron job to renew nexus cert based on certificate present in keyvault + content: | + #!/bin/bash + set -o errexit + echo "Calling configure-nexus-ssl script to renew ssl certificate" + bash /home/adminuser/configure-nexus-ssl.sh + permissions: '0755' + # Configure Nexus to disable default repos - path: /etc/nexus-data/etc/nexus.properties content: | From 8c05abb752ebc662f9f3af14851bf5d950b0dada Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 7 Apr 2022 13:10:52 +0000 Subject: [PATCH 059/142] Removed location references --- templates/shared_services/sonatype-nexus/terraform/vm.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 8ab94be297..b76c6ecd75 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -1,6 +1,6 @@ resource "azurerm_network_interface" "nexus" { name = "nic-nexus-${var.tre_id}" - location = var.location + location = data.azurerm_resource_group.rg.location resource_group_name = local.core_resource_group_name ip_configuration { @@ -72,7 +72,7 @@ resource "azurerm_key_vault_secret" "nexus_admin_password" { resource "azurerm_user_assigned_identity" "nexus_msi" { name = "id-nexus-${var.tre_id}" - location = var.location + location = data.azurerm_resource_group.rg.location resource_group_name = local.core_resource_group_name lifecycle { ignore_changes = [tags] } } @@ -94,7 +94,7 @@ resource "azurerm_key_vault_access_policy" "nexus_msi" { resource "azurerm_linux_virtual_machine" "nexus" { name = "nexus-${var.tre_id}" resource_group_name = local.core_resource_group_name - location = var.location + location = data.azurerm_resource_group.rg.location network_interface_ids = [azurerm_network_interface.nexus.id] size = "Standard_B2s" disable_password_authentication = false From d95349287b74589b8dcb6647592ab6b90693633d Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 7 Apr 2022 13:11:21 +0000 Subject: [PATCH 060/142] And another --- templates/shared_services/sonatype-nexus/terraform/vm.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index b76c6ecd75..b720640dc2 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -11,7 +11,7 @@ resource "azurerm_network_interface" "nexus" { } resource "azurerm_private_dns_zone" "nexus" { - name = "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" + name = "nexus-${var.tre_id}.${data.azurerm_resource_group.rg.location}.cloudapp.azure.com" resource_group_name = local.core_resource_group_name lifecycle { ignore_changes = [tags] } From e8210fbd7df1a6cdfa8e824cf99a9d9c0487daa8 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 11 Apr 2022 17:04:29 +0000 Subject: [PATCH 061/142] Removed location refs and added az cli --- .../shared_services/nexus-cert/Dockerfile.tmpl | 8 ++++++++ templates/shared_services/nexus-cert/porter.yaml | 5 ----- .../nexus-cert/template_schema.json | 9 +++++++++ .../nexus-cert/terraform/appgateway.tf | 16 ++++++++-------- .../nexus-cert/terraform/certificate.tf | 2 +- .../shared_services/nexus-cert/terraform/data.tf | 3 +++ .../nexus-cert/terraform/locals.tf | 2 -- .../nexus-cert/terraform/staticweb.tf | 4 ++-- .../nexus-cert/terraform/variables.tf | 1 - 9 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 templates/shared_services/nexus-cert/template_schema.json create mode 100644 templates/shared_services/nexus-cert/terraform/data.tf diff --git a/templates/shared_services/nexus-cert/Dockerfile.tmpl b/templates/shared_services/nexus-cert/Dockerfile.tmpl index 9a66e15f55..3f8524ce64 100755 --- a/templates/shared_services/nexus-cert/Dockerfile.tmpl +++ b/templates/shared_services/nexus-cert/Dockerfile.tmpl @@ -4,6 +4,14 @@ ARG BUNDLE_DIR RUN apt-get update && apt-get install -y ca-certificates +# Install Azure CLI +RUN apt-get update \ + && apt-get install -y ca-certificates jq curl apt-transport-https lsb-release gnupg \ + && curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null \ + && AZ_REPO=$(lsb_release -cs) \ + && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list \ + && apt-get update && apt-get -y install azure-cli + # This is a template Dockerfile for the bundle's invocation image # You can customize it to use different base images, install tools and copy configuration files. # diff --git a/templates/shared_services/nexus-cert/porter.yaml b/templates/shared_services/nexus-cert/porter.yaml index 383380af76..4e1ac969f1 100755 --- a/templates/shared_services/nexus-cert/porter.yaml +++ b/templates/shared_services/nexus-cert/porter.yaml @@ -19,9 +19,6 @@ parameters: - name: tre_id type: string description: "The ID of the parent TRE instance e.g., mytre-dev-3142" - - name: azure_location - type: string - description: "Azure location (region) to deploy to" - name: tfstate_resource_group_name type: string description: "Resource group containing the Terraform state storage account" @@ -44,7 +41,6 @@ install: input: false vars: tre_id: "{{ bundle.parameters.tre_id }}" - location: "{{ bundle.parameters.azure_location }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -68,7 +64,6 @@ uninstall: input: false vars: tre_id: "{{ bundle.parameters.tre_id }}" - location: "{{ bundle.parameters.azure_location }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" diff --git a/templates/shared_services/nexus-cert/template_schema.json b/templates/shared_services/nexus-cert/template_schema.json new file mode 100644 index 0000000000..c8e59956b8 --- /dev/null +++ b/templates/shared_services/nexus-cert/template_schema.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://github.com/microsoft/AzureTRE/templates/shared_services/sonatype-nexus/template_schema.json", + "type": "object", + "title": "Nexus Cert Service", + "description": "Provides SSL Cert for Nexus shared service", + "required": [], + "properties": {} +} diff --git a/templates/shared_services/nexus-cert/terraform/appgateway.tf b/templates/shared_services/nexus-cert/terraform/appgateway.tf index 4ca553d406..0a17ecd5a6 100644 --- a/templates/shared_services/nexus-cert/terraform/appgateway.tf +++ b/templates/shared_services/nexus-cert/terraform/appgateway.tf @@ -1,7 +1,7 @@ resource "azurerm_public_ip" "appgwpip" { name = "pip-nexus-${var.tre_id}" - resource_group_name = local.core_resource_group_name - location = var.location + resource_group_name = data.azurerm_resource_group.rg.name + location = data.azurerm_resource_group.rg.location allocation_method = "Static" sku = "Standard" domain_name_label = "nexus-${var.tre_id}" @@ -10,8 +10,8 @@ resource "azurerm_public_ip" "appgwpip" { } resource "azurerm_user_assigned_identity" "agw_id" { - resource_group_name = local.core_resource_group_name - location = var.location + resource_group_name = data.azurerm_resource_group.rg.name + location = data.azurerm_resource_group.rg.location name = "id-agw-nexuscert-${var.tre_id}" lifecycle { ignore_changes = [tags] } @@ -19,8 +19,8 @@ resource "azurerm_user_assigned_identity" "agw_id" { resource "azurerm_application_gateway" "agw" { name = "agw-nexuscert-${var.tre_id}" - resource_group_name = local.core_resource_group_name - location = var.location + resource_group_name = data.azurerm_resource_group.rg.name + location = data.azurerm_resource_group.rg.location sku { name = "Standard_v2" @@ -161,11 +161,11 @@ resource "azurerm_application_gateway" "agw" { data "azurerm_subnet" "app_gw_subnet" { name = "AppGwSubnet" virtual_network_name = "vnet-${var.tre_id}" - resource_group_name = local.core_resource_group_name + resource_group_name = data.azurerm_resource_group.rg.name } data "azurerm_public_ip" "appgwpip_data" { depends_on = [azurerm_application_gateway.agw] name = "pip-nexus-${var.tre_id}" - resource_group_name = local.core_resource_group_name + resource_group_name = data.azurerm_resource_group.rg.name } diff --git a/templates/shared_services/nexus-cert/terraform/certificate.tf b/templates/shared_services/nexus-cert/terraform/certificate.tf index 71d8afea68..05f1d989f1 100644 --- a/templates/shared_services/nexus-cert/terraform/certificate.tf +++ b/templates/shared_services/nexus-cert/terraform/certificate.tf @@ -45,5 +45,5 @@ resource "azurerm_key_vault_certificate" "tlscert" { data "azurerm_key_vault" "key_vault" { name = "kv-${var.tre_id}" - resource_group_name = local.core_resource_group_name + resource_group_name = data.azurerm_resource_group.rg.name } diff --git a/templates/shared_services/nexus-cert/terraform/data.tf b/templates/shared_services/nexus-cert/terraform/data.tf new file mode 100644 index 0000000000..45455479a0 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/data.tf @@ -0,0 +1,3 @@ +data "azurerm_resource_group" "rg" { + name = "rg-${var.tre_id}" +} diff --git a/templates/shared_services/nexus-cert/terraform/locals.tf b/templates/shared_services/nexus-cert/terraform/locals.tf index cc1d53f051..579394a4ff 100644 --- a/templates/shared_services/nexus-cert/terraform/locals.tf +++ b/templates/shared_services/nexus-cert/terraform/locals.tf @@ -1,8 +1,6 @@ locals { staticweb_storage_name = lower(replace("stwebnexus${var.tre_id}", "-", "")) - core_resource_group_name = "rg-${var.tre_id}" - staticweb_backend_pool_name = "beap-nexuscret-staticweb" app_path_map_name = "upm-nexuscert" redirect_path_map_name = "upm-nexuscert-redirect" diff --git a/templates/shared_services/nexus-cert/terraform/staticweb.tf b/templates/shared_services/nexus-cert/terraform/staticweb.tf index e5fc7829dc..de1285865a 100644 --- a/templates/shared_services/nexus-cert/terraform/staticweb.tf +++ b/templates/shared_services/nexus-cert/terraform/staticweb.tf @@ -3,8 +3,8 @@ data "azurerm_client_config" "deployer" {} # See https://microsoft.github.io/AzureTRE/tre-developers/letsencrypt/ resource "azurerm_storage_account" "staticweb" { name = local.staticweb_storage_name - resource_group_name = local.core_resource_group_name - location = var.location + resource_group_name = data.azurerm_resource_group.rg.name + location = data.azurerm_resource_group.rg.location account_kind = "StorageV2" account_tier = "Standard" account_replication_type = "LRS" diff --git a/templates/shared_services/nexus-cert/terraform/variables.tf b/templates/shared_services/nexus-cert/terraform/variables.tf index c60c7384d2..ac6838172f 100644 --- a/templates/shared_services/nexus-cert/terraform/variables.tf +++ b/templates/shared_services/nexus-cert/terraform/variables.tf @@ -1,3 +1,2 @@ variable "tre_id" {} -variable "location" {} From 833fc6825ef0a513ae34dd56e2beba0558f4cfdb Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 12 Apr 2022 17:09:56 +0000 Subject: [PATCH 062/142] Fixed nexus-cert kv permissions --- devops/scripts/check_dependencies.sh | 1 + resource_processor/shared/config.py | 3 ++ templates/core/terraform/outputs.sh | 1 + .../vmss_porter/cloud-config.yaml | 1 + .../resource_processor/vmss_porter/main.tf | 4 ++- .../nexus-cert/Dockerfile.tmpl | 0 .../shared_services/nexus-cert/azure.json | 32 ------------------- .../shared_services/nexus-cert/porter.yaml | 14 ++++++-- .../nexus-cert/terraform/appgateway.tf | 14 ++------ .../nexus-cert/terraform/certificate.tf | 13 ++------ .../nexus-cert/terraform/data.tf | 17 ++++++++++ .../nexus-cert/terraform/staticweb.tf | 4 +-- .../nexus-cert/terraform/variables.tf | 7 ++++ 13 files changed, 50 insertions(+), 61 deletions(-) mode change 100755 => 100644 templates/shared_services/nexus-cert/Dockerfile.tmpl delete mode 100755 templates/shared_services/nexus-cert/azure.json diff --git a/devops/scripts/check_dependencies.sh b/devops/scripts/check_dependencies.sh index 4bf102f0b4..67eff47a13 100755 --- a/devops/scripts/check_dependencies.sh +++ b/devops/scripts/check_dependencies.sh @@ -59,6 +59,7 @@ fi export SUB_NAME=$(az account show --query name -o tsv) export SUB_ID=$(az account show --query id -o tsv) export TENANT_ID=$(az account show --query tenantId -o tsv) +export OBJECT_ID=$(az ad signed-in-user show --query objectId -o tsv) if [ -z "$SUB_NAME" ]; then echo -e "\n\e[31m»»» ⚠️ You are not logged in to Azure!" exit 1 diff --git a/resource_processor/shared/config.py b/resource_processor/shared/config.py index 63f5fa6dcb..645539f1f4 100644 --- a/resource_processor/shared/config.py +++ b/resource_processor/shared/config.py @@ -26,6 +26,9 @@ def get_config(logger_adapter) -> dict: config["arm_client_id"] = os.environ["ARM_CLIENT_ID"] config["arm_tenant_id"] = os.environ["AZURE_TENANT_ID"] + # Needed for deploying bundles that need deployer's object Id + config["arm_object_id"] = os.environ["AZURE_OBJECT_ID"] + # Only set client secret if MSI is disabled config["arm_client_secret"] = os.environ["ARM_CLIENT_SECRET"] if config["arm_use_msi"] == "false" else "" diff --git a/templates/core/terraform/outputs.sh b/templates/core/terraform/outputs.sh index 912f06627e..72bc46d49d 100755 --- a/templates/core/terraform/outputs.sh +++ b/templates/core/terraform/outputs.sh @@ -27,3 +27,4 @@ echo "TEST_WORKSPACE_APP_ID='${WORKSPACE_API_CLIENT_ID}'" >> ../private.env echo "SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env echo "AZURE_SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env echo "AZURE_TENANT_ID='${TENANT_ID}'" >> ../private.env +echo "AZURE_OBJECT_ID='${OBJECT_ID}'" >> ../private.env diff --git a/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml b/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml index a42911098b..22d0afb23c 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml +++ b/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml @@ -36,6 +36,7 @@ write_files: VMSS_MSI_ID=${vmss_msi_id} AZURE_SUBSCRIPTION_ID=${arm_subscription_id} ARM_CLIENT_ID=${vmss_msi_id} + AZURE_OBJECT_ID=${arm_object_id} AZURE_TENANT_ID=${arm_tenant_id} ARM_USE_MSI=true APPLICATIONINSIGHTS_CONNECTION_STRING=${app_insights_connection_string} diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index 2facd5068c..4ace3a5a20 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -15,6 +15,7 @@ data "template_file" "cloudconfig" { vmss_msi_id = azurerm_user_assigned_identity.vmss_msi.client_id arm_subscription_id = data.azurerm_subscription.current.subscription_id arm_tenant_id = data.azurerm_client_config.current.tenant_id + arm_object_id = azurerm_user_assigned_identity.vmss_msi.principal_id resource_processor_vmss_porter_image_repository = var.resource_processor_vmss_porter_image_repository resource_processor_vmss_porter_image_tag = local.version app_insights_connection_string = var.app_insights_connection_string @@ -169,5 +170,6 @@ resource "azurerm_key_vault_access_policy" "resource_processor" { tenant_id = azurerm_user_assigned_identity.vmss_msi.tenant_id object_id = azurerm_user_assigned_identity.vmss_msi.principal_id - secret_permissions = ["Get", "List", "Set", "Delete"] + secret_permissions = ["Get", "List", "Set", "Delete"] + certificate_permissions = ["Get", "Update", "Create", "Import", "Delete"] } diff --git a/templates/shared_services/nexus-cert/Dockerfile.tmpl b/templates/shared_services/nexus-cert/Dockerfile.tmpl old mode 100755 new mode 100644 diff --git a/templates/shared_services/nexus-cert/azure.json b/templates/shared_services/nexus-cert/azure.json deleted file mode 100755 index cdc4c1365c..0000000000 --- a/templates/shared_services/nexus-cert/azure.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "schemaVersion": "1.0.0-DRAFT+b6c701f", - "name": "azure", - "created": "2021-06-03T11:31:05.7314113Z", - "modified": "2021-06-03T11:31:05.7314113Z", - "credentials": [ - { - "name": "azure_client_id", - "source": { - "env": "ARM_CLIENT_ID" - } - }, - { - "name": "azure_client_secret", - "source": { - "env": "ARM_CLIENT_SECRET" - } - }, - { - "name": "azure_subscription_id", - "source": { - "env": "ARM_SUBSCRIPTION_ID" - } - }, - { - "name": "azure_tenant_id", - "source": { - "env": "ARM_TENANT_ID" - } - } - ] -} diff --git a/templates/shared_services/nexus-cert/porter.yaml b/templates/shared_services/nexus-cert/porter.yaml index 4e1ac969f1..1b7eb20fd7 100755 --- a/templates/shared_services/nexus-cert/porter.yaml +++ b/templates/shared_services/nexus-cert/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-nexus-cert -version: 0.0.1 +version: 0.0.8 description: "An Azure TRE Nexus certificate creation shared service" registry: azuretre dockerfile: Dockerfile.tmpl @@ -29,6 +29,13 @@ parameters: type: string default: "tfstate" description: "The name of the Terraform state storage container" + - name: arm_use_msi + env: ARM_USE_MSI + type: boolean + default: false + - name: arm_object_id + env: AZURE_OBJECT_ID + type: string mixins: - exec @@ -41,6 +48,7 @@ install: input: false vars: tre_id: "{{ bundle.parameters.tre_id }}" + deployer_object_id: "{{ bundle.parameters.arm_object_id }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -49,7 +57,7 @@ install: container_name: "{{ bundle.parameters.tfstate_container_name }}" key: - "{{ bundle.parameters.tre_id }}-shared-service-firewall" + "{{ bundle.parameters.tre_id }}-shared-service-nexus-cert" upgrade: - exec: @@ -72,4 +80,4 @@ uninstall: container_name: "{{ bundle.parameters.tfstate_container_name }}" key: - "{{ bundle.parameters.tre_id }}-shared-service-firewall" + "{{ bundle.parameters.tre_id }}-shared-service-nexus-cert" diff --git a/templates/shared_services/nexus-cert/terraform/appgateway.tf b/templates/shared_services/nexus-cert/terraform/appgateway.tf index 0a17ecd5a6..a0be40a584 100644 --- a/templates/shared_services/nexus-cert/terraform/appgateway.tf +++ b/templates/shared_services/nexus-cert/terraform/appgateway.tf @@ -156,16 +156,8 @@ resource "azurerm_application_gateway" "agw" { ] } -} - -data "azurerm_subnet" "app_gw_subnet" { - name = "AppGwSubnet" - virtual_network_name = "vnet-${var.tre_id}" - resource_group_name = data.azurerm_resource_group.rg.name -} + depends_on = [ + azurerm_key_vault_access_policy.app_gw_managed_identity + ] -data "azurerm_public_ip" "appgwpip_data" { - depends_on = [azurerm_application_gateway.agw] - name = "pip-nexus-${var.tre_id}" - resource_group_name = data.azurerm_resource_group.rg.name } diff --git a/templates/shared_services/nexus-cert/terraform/certificate.tf b/templates/shared_services/nexus-cert/terraform/certificate.tf index 05f1d989f1..938354c730 100644 --- a/templates/shared_services/nexus-cert/terraform/certificate.tf +++ b/templates/shared_services/nexus-cert/terraform/certificate.tf @@ -3,13 +3,8 @@ resource "azurerm_key_vault_access_policy" "app_gw_managed_identity" { tenant_id = azurerm_user_assigned_identity.agw_id.tenant_id object_id = azurerm_user_assigned_identity.agw_id.principal_id - key_permissions = [ - "Get", - ] - - secret_permissions = [ - "Get", - ] + key_permissions = ["Get"] + secret_permissions = ["Get"] } resource "azurerm_key_vault_certificate" "tlscert" { @@ -41,9 +36,5 @@ resource "azurerm_key_vault_certificate" "tlscert" { lifecycle { ignore_changes = all } -} -data "azurerm_key_vault" "key_vault" { - name = "kv-${var.tre_id}" - resource_group_name = data.azurerm_resource_group.rg.name } diff --git a/templates/shared_services/nexus-cert/terraform/data.tf b/templates/shared_services/nexus-cert/terraform/data.tf index 45455479a0..2d089e2019 100644 --- a/templates/shared_services/nexus-cert/terraform/data.tf +++ b/templates/shared_services/nexus-cert/terraform/data.tf @@ -1,3 +1,20 @@ data "azurerm_resource_group" "rg" { name = "rg-${var.tre_id}" } + +data "azurerm_key_vault" "key_vault" { + name = "kv-${var.tre_id}" + resource_group_name = data.azurerm_resource_group.rg.name +} + +data "azurerm_subnet" "app_gw_subnet" { + name = "AppGwSubnet" + virtual_network_name = "vnet-${var.tre_id}" + resource_group_name = data.azurerm_resource_group.rg.name +} + +data "azurerm_public_ip" "appgwpip_data" { + depends_on = [azurerm_application_gateway.agw] + name = "pip-nexus-${var.tre_id}" + resource_group_name = data.azurerm_resource_group.rg.name +} diff --git a/templates/shared_services/nexus-cert/terraform/staticweb.tf b/templates/shared_services/nexus-cert/terraform/staticweb.tf index de1285865a..edebfa590b 100644 --- a/templates/shared_services/nexus-cert/terraform/staticweb.tf +++ b/templates/shared_services/nexus-cert/terraform/staticweb.tf @@ -1,5 +1,3 @@ -data "azurerm_client_config" "deployer" {} - # See https://microsoft.github.io/AzureTRE/tre-developers/letsencrypt/ resource "azurerm_storage_account" "staticweb" { name = local.staticweb_storage_name @@ -27,5 +25,5 @@ resource "azurerm_storage_account" "staticweb" { resource "azurerm_role_assignment" "stgwriter" { scope = azurerm_storage_account.staticweb.id role_definition_name = "Storage Blob Data Contributor" - principal_id = data.azurerm_client_config.deployer.object_id + principal_id = var.deployer_object_id } diff --git a/templates/shared_services/nexus-cert/terraform/variables.tf b/templates/shared_services/nexus-cert/terraform/variables.tf index ac6838172f..ef932260db 100644 --- a/templates/shared_services/nexus-cert/terraform/variables.tf +++ b/templates/shared_services/nexus-cert/terraform/variables.tf @@ -1,2 +1,9 @@ variable "tre_id" {} + +// We have to inject this via variable instead of getting via conventional +// data.azurerm_client_config.current.object_id due to a bug with TF & MSI +// https://github.com/hashicorp/terraform-provider-azurerm/issues/7787 +variable "deployer_object_id" { + type = string +} From c28500851381a50396c6f26cc8431c0778b4eedc Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 12 Apr 2022 20:36:33 +0000 Subject: [PATCH 063/142] Corrected outputs directory --- templates/shared_services/nexus-cert/scripts/outputs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/shared_services/nexus-cert/scripts/outputs.sh b/templates/shared_services/nexus-cert/scripts/outputs.sh index 11b11a4114..d84a89531a 100755 --- a/templates/shared_services/nexus-cert/scripts/outputs.sh +++ b/templates/shared_services/nexus-cert/scripts/outputs.sh @@ -3,6 +3,7 @@ set -e if [ ! -f ../tre_output.json ]; then # Connect to the remote backend of Terraform + pushd ../terraform > /dev/null export TF_LOG="" terraform init -input=false -backend=true -reconfigure -upgrade \ -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name:?}" \ @@ -12,6 +13,7 @@ if [ ! -f ../tre_output.json ]; then # Convert the output to json terraform output -json > ../tre_output.json + popd > /dev/null fi # Now create an .env file From 329482a5ddf8c77f0650fd2a628324e0f6d92d79 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 13 Apr 2022 13:02:09 +0000 Subject: [PATCH 064/142] Fixed shared service deployment steps --- devops/scripts/check_dependencies.sh | 2 -- resource_processor/shared/config.py | 3 --- templates/core/terraform/outputs.sh | 1 - .../resource_processor/vmss_porter/cloud-config.yaml | 1 - .../terraform/resource_processor/vmss_porter/main.tf | 1 - templates/shared_services/nexus-cert/.dockerignore | 3 +++ templates/shared_services/nexus-cert/porter.yaml | 6 +----- .../shared_services/nexus-cert/scripts/auth-hook.sh | 2 +- .../nexus-cert/scripts/json-to-env.sh | 8 ++++++-- .../nexus-cert/scripts/letsencrypt.sh | 12 +++++++++--- .../shared_services/nexus-cert/scripts/outputs.sh | 2 +- .../shared_services/nexus-cert/terraform/output.tf | 6 +++++- .../nexus-cert/terraform/staticweb.tf | 7 ------- .../nexus-cert/terraform/variables.tf | 8 -------- 14 files changed, 26 insertions(+), 36 deletions(-) diff --git a/devops/scripts/check_dependencies.sh b/devops/scripts/check_dependencies.sh index ca22699168..d185809b00 100755 --- a/devops/scripts/check_dependencies.sh +++ b/devops/scripts/check_dependencies.sh @@ -54,8 +54,6 @@ SUB_ID=$(az account show --query id -o tsv) export SUB_ID TENANT_ID=$(az account show --query tenantId -o tsv) export TENANT_ID -OBJECT_ID=$(az ad signed-in-user show --query objectId -o tsv) -export OBJECT_ID if [ -z "$SUB_NAME" ]; then echo -e "\n\e[31m»»» ⚠️ You are not logged in to Azure!" diff --git a/resource_processor/shared/config.py b/resource_processor/shared/config.py index 645539f1f4..63f5fa6dcb 100644 --- a/resource_processor/shared/config.py +++ b/resource_processor/shared/config.py @@ -26,9 +26,6 @@ def get_config(logger_adapter) -> dict: config["arm_client_id"] = os.environ["ARM_CLIENT_ID"] config["arm_tenant_id"] = os.environ["AZURE_TENANT_ID"] - # Needed for deploying bundles that need deployer's object Id - config["arm_object_id"] = os.environ["AZURE_OBJECT_ID"] - # Only set client secret if MSI is disabled config["arm_client_secret"] = os.environ["ARM_CLIENT_SECRET"] if config["arm_use_msi"] == "false" else "" diff --git a/templates/core/terraform/outputs.sh b/templates/core/terraform/outputs.sh index 72bc46d49d..912f06627e 100755 --- a/templates/core/terraform/outputs.sh +++ b/templates/core/terraform/outputs.sh @@ -27,4 +27,3 @@ echo "TEST_WORKSPACE_APP_ID='${WORKSPACE_API_CLIENT_ID}'" >> ../private.env echo "SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env echo "AZURE_SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env echo "AZURE_TENANT_ID='${TENANT_ID}'" >> ../private.env -echo "AZURE_OBJECT_ID='${OBJECT_ID}'" >> ../private.env diff --git a/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml b/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml index 22d0afb23c..a42911098b 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml +++ b/templates/core/terraform/resource_processor/vmss_porter/cloud-config.yaml @@ -36,7 +36,6 @@ write_files: VMSS_MSI_ID=${vmss_msi_id} AZURE_SUBSCRIPTION_ID=${arm_subscription_id} ARM_CLIENT_ID=${vmss_msi_id} - AZURE_OBJECT_ID=${arm_object_id} AZURE_TENANT_ID=${arm_tenant_id} ARM_USE_MSI=true APPLICATIONINSIGHTS_CONNECTION_STRING=${app_insights_connection_string} diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index 4ace3a5a20..3624a719d3 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -15,7 +15,6 @@ data "template_file" "cloudconfig" { vmss_msi_id = azurerm_user_assigned_identity.vmss_msi.client_id arm_subscription_id = data.azurerm_subscription.current.subscription_id arm_tenant_id = data.azurerm_client_config.current.tenant_id - arm_object_id = azurerm_user_assigned_identity.vmss_msi.principal_id resource_processor_vmss_porter_image_repository = var.resource_processor_vmss_porter_image_repository resource_processor_vmss_porter_image_tag = local.version app_insights_connection_string = var.app_insights_connection_string diff --git a/templates/shared_services/nexus-cert/.dockerignore b/templates/shared_services/nexus-cert/.dockerignore index 2919244c86..852f29463e 100644 --- a/templates/shared_services/nexus-cert/.dockerignore +++ b/templates/shared_services/nexus-cert/.dockerignore @@ -1,4 +1,7 @@ # See https://docs.docker.com/engine/reference/builder/#dockerignore-file # Put files here that you don't want copied into your bundle's invocation image .gitignore +**/.terraform/* +**/.terraform.lock.hcl +**/*_backend.tf Dockerfile.tmpl diff --git a/templates/shared_services/nexus-cert/porter.yaml b/templates/shared_services/nexus-cert/porter.yaml index 1b7eb20fd7..4160d241fe 100755 --- a/templates/shared_services/nexus-cert/porter.yaml +++ b/templates/shared_services/nexus-cert/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-nexus-cert -version: 0.0.8 +version: 0.0.9 description: "An Azure TRE Nexus certificate creation shared service" registry: azuretre dockerfile: Dockerfile.tmpl @@ -33,9 +33,6 @@ parameters: env: ARM_USE_MSI type: boolean default: false - - name: arm_object_id - env: AZURE_OBJECT_ID - type: string mixins: - exec @@ -48,7 +45,6 @@ install: input: false vars: tre_id: "{{ bundle.parameters.tre_id }}" - deployer_object_id: "{{ bundle.parameters.arm_object_id }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" diff --git a/templates/shared_services/nexus-cert/scripts/auth-hook.sh b/templates/shared_services/nexus-cert/scripts/auth-hook.sh index ea87070267..e25a0eea30 100755 --- a/templates/shared_services/nexus-cert/scripts/auth-hook.sh +++ b/templates/shared_services/nexus-cert/scripts/auth-hook.sh @@ -6,7 +6,7 @@ EOF # shellcheck disable=SC2016 az storage blob upload \ - --account-name "${STORAGE_ACCOUNT}" \ + --account-name "${STORAGE_ACCOUNT_NAME}" \ --auth-mode login \ --container-name '$web' \ --file 'validation.txt' \ diff --git a/templates/shared_services/nexus-cert/scripts/json-to-env.sh b/templates/shared_services/nexus-cert/scripts/json-to-env.sh index 923c4fc193..5926e4edb1 100755 --- a/templates/shared_services/nexus-cert/scripts/json-to-env.sh +++ b/templates/shared_services/nexus-cert/scripts/json-to-env.sh @@ -14,8 +14,12 @@ jq -r ' "env_var": "APPLICATION_GATEWAY" }, { - "path": "storage_account", - "env_var": "STORAGE_ACCOUNT" + "path": "storage_account_id", + "env_var": "STORAGE_ACCOUNT_ID" + }, + { + "path": "storage_account_name", + "env_var": "STORAGE_ACCOUNT_NAME" }, { "path": "resource_group_name", diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh index 0bf013a262..09b550a5c2 100755 --- a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh +++ b/templates/shared_services/nexus-cert/scripts/letsencrypt.sh @@ -2,13 +2,19 @@ set -e script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") -if [[ -z ${STORAGE_ACCOUNT} ]]; then +if [[ -z ${STORAGE_ACCOUNT_NAME} ]]; then echo "STORAGE_ACCOUNT not set" exit 1 fi echo "Checking for index.html file in storage account" +# Assign Storage Blob Data Contributor permissions if not already present +objectId=$(az ad signed-in-user show --query objectId -o tsv) +az role assignment create --assignee "${objectId}" \ + --role "Storage Blob Data Contributor" \ + --scope "${STORAGE_ACCOUNT_ID}" + # Create the default index.html page cat << EOF > index.html @@ -17,7 +23,7 @@ EOF # shellcheck disable=SC2016 indexExists=$(az storage blob list -o json \ - --account-name "${STORAGE_ACCOUNT}" \ + --account-name "${STORAGE_ACCOUNT_NAME}" \ --auth-mode login \ --container-name '$web' \ --query "[?name=='index.html'].name" \ @@ -28,7 +34,7 @@ if [[ ${indexExists} -lt 1 ]]; then # shellcheck disable=SC2016 az storage blob upload \ - --account-name "${STORAGE_ACCOUNT}" \ + --account-name "${STORAGE_ACCOUNT_NAME}" \ --auth-mode login \ --container-name '$web' \ --file index.html \ diff --git a/templates/shared_services/nexus-cert/scripts/outputs.sh b/templates/shared_services/nexus-cert/scripts/outputs.sh index d84a89531a..753d63b672 100755 --- a/templates/shared_services/nexus-cert/scripts/outputs.sh +++ b/templates/shared_services/nexus-cert/scripts/outputs.sh @@ -9,7 +9,7 @@ if [ ! -f ../tre_output.json ]; then -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name:?}" \ -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name:?}" \ -backend-config="container_name=${TF_VAR_terraform_state_container_name:?}" \ - -backend-config="key=${TRE_ID}" + -backend-config="key=${TRE_ID}-shared-service-nexus-cert" # Convert the output to json terraform output -json > ../tre_output.json diff --git a/templates/shared_services/nexus-cert/terraform/output.tf b/templates/shared_services/nexus-cert/terraform/output.tf index 30faf2a47a..d8a4b8d2b3 100644 --- a/templates/shared_services/nexus-cert/terraform/output.tf +++ b/templates/shared_services/nexus-cert/terraform/output.tf @@ -6,10 +6,14 @@ output "application_gateway" { value = azurerm_application_gateway.agw.name } -output "storage_account" { +output "storage_account_name" { value = azurerm_storage_account.staticweb.name } +output "storage_account_id" { + value = azurerm_storage_account.staticweb.id +} + output "resource_group_name" { value = azurerm_application_gateway.agw.resource_group_name } diff --git a/templates/shared_services/nexus-cert/terraform/staticweb.tf b/templates/shared_services/nexus-cert/terraform/staticweb.tf index edebfa590b..d882900533 100644 --- a/templates/shared_services/nexus-cert/terraform/staticweb.tf +++ b/templates/shared_services/nexus-cert/terraform/staticweb.tf @@ -20,10 +20,3 @@ resource "azurerm_storage_account" "staticweb" { lifecycle { ignore_changes = [tags] } } - -# Assign the "Storage Blob Data Contributor" role needed for uploading certificates to the storage account -resource "azurerm_role_assignment" "stgwriter" { - scope = azurerm_storage_account.staticweb.id - role_definition_name = "Storage Blob Data Contributor" - principal_id = var.deployer_object_id -} diff --git a/templates/shared_services/nexus-cert/terraform/variables.tf b/templates/shared_services/nexus-cert/terraform/variables.tf index ef932260db..798db79a9f 100644 --- a/templates/shared_services/nexus-cert/terraform/variables.tf +++ b/templates/shared_services/nexus-cert/terraform/variables.tf @@ -1,9 +1 @@ - variable "tre_id" {} - -// We have to inject this via variable instead of getting via conventional -// data.azurerm_client_config.current.object_id due to a bug with TF & MSI -// https://github.com/hashicorp/terraform-provider-azurerm/issues/7787 -variable "deployer_object_id" { - type = string -} From 69f3acfbc4969b6bae2bbbfdc0a6155a10f38feb Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 13 Apr 2022 17:46:31 +0000 Subject: [PATCH 065/142] Updated docs and removed renew prompt --- .../setup-instructions/configuring-shared-services.md | 8 +++++--- .../sonatype-nexus/terraform/cloud-config.yaml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index ed86f690fa..4b0846755b 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -12,12 +12,14 @@ Complete the configuration of the shared services (Nexus and Gitea) from inside 6. Git clone the TRE repository: ```git clone https://github.com/microsoft/AzureTRE.git``` 7. Download jq ```curl -L -o /usr/bin/jq.exe https://github.com/stedolan/jq/releases/latest/download/jq-win64.exe``` -## Configure Nexus repository +## Configure Nexus repository proxies -1. Run the Nexus configuration script to reset the password and setup a PyPI proxy on Nexus: +1. Run the Nexus configuration script to reset the password and set up several common repository proxies on Nexus. Substitute `` with the TRE_ID you chose for the core deployment and `` with the Azure region you deployed to: ```./templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh -t -l ``` -## Configure Gitea repository +You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/` in the jumpbox and signing in with the username `admin` and the password secret located in your core keyvault, with the key `nexus-admin-password`. + +## Configure Gitea repositories Note : This is a Gitea *shared service* which will be accessible from all workspaces intended for mirroring external Git repositories. A Gitea *workspace service* can also be deployed per workspace to enable Gitea to be used within a specific workspace. diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index aaa1402617..7e01547a96 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -91,7 +91,7 @@ write_files: ((keystore_timeout--)) done echo 'Directory found. Importing ssl cert into nexus-data/keystores/keystore.jks...' - keytool -v -importkeystore -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 \ + keytool -v -importkeystore -noprompt -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 \ -destkeystore /etc/nexus-data/keystores/keystore.jks \ -deststoretype JKS -srcstorepass "$CERT_PASSWORD" -deststorepass "$CERT_PASSWORD" From 2e494c6c13a15f1f78fe89f0028b6004f69833e0 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 13 Apr 2022 17:48:34 +0000 Subject: [PATCH 066/142] version bump --- templates/shared_services/gitea/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/gitea/version.txt b/templates/shared_services/gitea/version.txt index 0a8da88258..f1380eede2 100644 --- a/templates/shared_services/gitea/version.txt +++ b/templates/shared_services/gitea/version.txt @@ -1 +1 @@ -__version__ = "0.1.6" +__version__ = "0.1.7" From 407916ccc960c45f430e159bd6a62ec71a9ed57f Mon Sep 17 00:00:00 2001 From: marrobi Date: Thu, 14 Apr 2022 10:39:58 +0000 Subject: [PATCH 067/142] Increase bundle versions --- .../user_resources/guacamole-azure-linuxvm/porter.yaml | 2 +- .../user_resources/guacamole-azure-windowsvm/porter.yaml | 2 +- templates/workspaces/base/porter.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index 3a7b39621b..8c5874b840 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-linuxvm -version: 0.1.10 +version: 0.1.11 description: "An Azure TRE User Resource Template for Guacamole (Linux)" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index 9b672271ce..a66d10c39a 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-windowsvm -version: 0.1.8 +version: 0.1.9 description: "An Azure TRE User Resource Template for Guacamole (Windows 10)" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index d4eeb5e4dc..16d582b8d2 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-base -version: 0.1.13 +version: 0.1.14 description: "A base Azure TRE workspace" registry: azuretre From baceac4f1b12bb97266bb5db3177d4b470f5eb5c Mon Sep 17 00:00:00 2001 From: marrobi Date: Thu, 14 Apr 2022 11:48:25 +0000 Subject: [PATCH 068/142] remote location from variables files --- .../guacamole-azure-linuxvm/terraform/variables.tf | 1 - .../guacamole-azure-windowsvm/terraform/variables.tf | 1 - 2 files changed, 2 deletions(-) diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf index f3d4198374..11b9a24565 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf @@ -1,7 +1,6 @@ variable "workspace_id" {} variable "tre_id" {} variable "parent_service_id" {} -variable "location" {} variable "arm_client_id" {} variable "arm_client_secret" {} diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf index f3d4198374..11b9a24565 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf @@ -1,7 +1,6 @@ variable "workspace_id" {} variable "tre_id" {} variable "parent_service_id" {} -variable "location" {} variable "arm_client_id" {} variable "arm_client_secret" {} From 7a271618997e6dca3f06e2e76b9fb467f5cded39 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 25 Apr 2022 22:45:15 +0000 Subject: [PATCH 069/142] Removed shared service make --- Makefile | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Makefile b/Makefile index 5dc3d53bdd..5467200f47 100644 --- a/Makefile +++ b/Makefile @@ -156,16 +156,6 @@ prepare-tf-state: && pushd ./templates/core/terraform > /dev/null && ../../shared_services/firewall/terraform/remove_state.sh && popd > /dev/null \ && pushd ./templates/shared_services/firewall/terraform > /dev/null && ./import_state.sh && popd > /dev/null -terraform-shared-service-deploy: - $(call target_title, "Deploying ${DIR} with Terraform") \ - && . ./devops/scripts/check_dependencies.sh \ - && . ./devops/scripts/load_env.sh ./templates/core/.env \ - && . ./devops/scripts/load_env.sh ./devops/.env \ - && . ./devops/scripts/load_terraform_env.sh ./devops/.env \ - && . ./devops/scripts/load_terraform_env.sh ./templates/core/.env \ - && . ./devops/scripts/key_vault_list.sh \ - && if [[ "$${TF_LOG}" == "DEBUG" ]]; then echo "TF DEBUG set - output supressed - see tflogs container for log file" && cd ${DIR} && ../../deploy_from_local.sh 1>/dev/null 2>/dev/null; else cd ${DIR} && ../../deploy_from_local.sh; fi; - # / End migration targets nexus-letsencrypt: From e07205c25f51179e6a6bbe4e5fed61da631c6e1a Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 25 Apr 2022 22:46:34 +0000 Subject: [PATCH 070/142] Removed docker prune --- .../sonatype-nexus/terraform/cloud-config.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 7e01547a96..ca380769e3 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -35,20 +35,6 @@ system_info: groups: [docker] write_files: - - path: /etc/cron.hourly/docker-prune - # An hourly cron job to have docker free disk space. Running this frquently - # since disk might get full fast, but we prune only when free space is low. - content: | - #!/bin/bash - set -o errexit - used_percent=$(df / --output=pcent | tail -1 | sed 's/[^0-9]//g') - echo "Used disk space percent: $${used_percent}" - if (( used_percent > 60 )); then - echo "Free space too low, pruning..." - docker system prune -f - fi - permissions: '0755' - - path: /etc/cron.daily/renew-nexus-cert # Daily cron job to renew nexus cert based on certificate present in keyvault content: | From ee40232e8b36cb52cbc1454593fbd52c3ce0fa70 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 25 Apr 2022 22:55:23 +0000 Subject: [PATCH 071/142] Bash headers --- .../sonatype-nexus/terraform/cloud-config.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index ca380769e3..37fd130f99 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -53,6 +53,12 @@ write_files: # Set up Nexus to serve https using SSL cert - path: /home/adminuser/configure-nexus-ssl.sh content: | + #!/bin/bash + set -o errexit + set -o pipefail + set -o nounset + # set -o xtrace + # Prepare ssl certificate az login --identity -u ${msi_id} # -- get cert from kv as secret so it contains private key @@ -119,6 +125,11 @@ write_files: - path: /home/adminuser/reset-nexus-password.sh content: | #!/bin/bash + set -o errexit + set -o pipefail + set -o nounset + # set -o xtrace + # Get the current password so we can post to the API # (this is created in /nexus-data mounted volume as part of Nexus container start-up) password_timeout=300 From 561379c2a97fa7ab1db287382ac5995e3f34b56c Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 25 Apr 2022 22:59:17 +0000 Subject: [PATCH 072/142] Layer clean --- templates/shared_services/nexus-cert/Dockerfile.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/shared_services/nexus-cert/Dockerfile.tmpl b/templates/shared_services/nexus-cert/Dockerfile.tmpl index 3f8524ce64..c7e7f95586 100644 --- a/templates/shared_services/nexus-cert/Dockerfile.tmpl +++ b/templates/shared_services/nexus-cert/Dockerfile.tmpl @@ -11,6 +11,7 @@ RUN apt-get update \ && AZ_REPO=$(lsb_release -cs) \ && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list \ && apt-get update && apt-get -y install azure-cli + && apt-get clean -y && rm -rf /var/lib/apt/lists/* # This is a template Dockerfile for the bundle's invocation image # You can customize it to use different base images, install tools and copy configuration files. From 84c9349f3f7c001d43ce0c3959790daac7a91bc6 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 25 Apr 2022 23:01:35 +0000 Subject: [PATCH 073/142] Reduce layer --- templates/shared_services/nexus-cert/Dockerfile.tmpl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/shared_services/nexus-cert/Dockerfile.tmpl b/templates/shared_services/nexus-cert/Dockerfile.tmpl index c7e7f95586..0832adde5a 100644 --- a/templates/shared_services/nexus-cert/Dockerfile.tmpl +++ b/templates/shared_services/nexus-cert/Dockerfile.tmpl @@ -2,7 +2,9 @@ FROM debian:stretch-slim ARG BUNDLE_DIR -RUN apt-get update && apt-get install -y ca-certificates +RUN apt-get update \ + && apt-get install -y ca-certificates \ + && apt-get clean -y && rm -rf /var/lib/apt/lists/* # Install Azure CLI RUN apt-get update \ @@ -10,7 +12,7 @@ RUN apt-get update \ && curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null \ && AZ_REPO=$(lsb_release -cs) \ && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list \ - && apt-get update && apt-get -y install azure-cli + && apt-get update && apt-get -y install azure-cli \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* # This is a template Dockerfile for the bundle's invocation image From 7213972ec687e82206a4048a09f14097c4abeb67 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 26 Apr 2022 13:52:14 +0000 Subject: [PATCH 074/142] Testing without kv role assignment --- Makefile | 5 ++-- templates/core/terraform/.terraform.lock.hcl | 26 +++++++++---------- .../sonatype-nexus/terraform/vm.tf | 10 +++---- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 5467200f47..8777432ab9 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ firewall-install: && $(MAKE) bundle-register DIR="./templates/shared_services/firewall" BUNDLE_TYPE=shared_service \ && $(MAKE) deploy-shared-service DIR=./templates/shared_services/firewall/ BUNDLE_TYPE=shared_service -nexus-install: nexus-cert-install nexus-letsencrypt +nexus-install: $(MAKE) bundle-build DIR=./templates/shared_services/sonatype-nexus/ \ && $(MAKE) bundle-publish DIR=./templates/shared_services/sonatype-nexus/ \ && $(MAKE) bundle-register DIR="./templates/shared_services/sonatype-nexus" BUNDLE_TYPE=shared_service \ @@ -109,7 +109,8 @@ gitea-install: nexus-cert-install: $(MAKE) bundle-build DIR=./templates/shared_services/nexus-cert/ \ && $(MAKE) bundle-publish DIR=./templates/shared_services/nexus-cert/ \ - && $(MAKE) shared-service-register-and-deploy DIR=./templates/shared_services/nexus-cert/ BUNDLE_TYPE=shared_service + && $(MAKE) bundle-register DIR="./templates/shared_services/nexus-cert" BUNDLE_TYPE=shared_service \ + && $(MAKE) deploy-shared-service DIR=./templates/shared_services/nexus-cert/ BUNDLE_TYPE=shared_service # A recipe for pushing images. Parameters: # 1. Image name suffix diff --git a/templates/core/terraform/.terraform.lock.hcl b/templates/core/terraform/.terraform.lock.hcl index 21a46d25a4..61aa39d650 100644 --- a/templates/core/terraform/.terraform.lock.hcl +++ b/templates/core/terraform/.terraform.lock.hcl @@ -58,21 +58,21 @@ provider "registry.terraform.io/hashicorp/local" { } provider "registry.terraform.io/hashicorp/random" { - version = "3.1.2" + version = "3.1.3" hashes = [ - "h1:5A5VsY5wNmOZlupUcLnIoziMPn8htSZBXbP3lI7lBEM=", - "zh:0daceba867b330d3f8e2c5dc895c4291845a78f31955ce1b91ab2c4d1cd1c10b", - "zh:104050099efd30a630741f788f9576b19998e7a09347decbec3da0b21d64ba2d", - "zh:173f4ef3fdf0c7e2564a3db0fac560e9f5afdf6afd0b75d6646af6576b122b16", - "zh:41d50f975e535f968b3f37170fb07937c15b76d85ba947d0ce5e5ff9530eda65", - "zh:51a5038867e5e60757ed7f513dd6a973068241190d158a81d1b69296efb9cb8d", - "zh:6432a568e97a5a36cc8aebca5a7e9c879a55d3bc71d0da1ab849ad905f41c0be", - "zh:6bac6501394b87138a5e17c9f3a41e46ff7833ad0ba2a96197bb7787e95b641c", - "zh:6c0a7f5faacda644b022e7718e53f5868187435be6d000786d1ca05aa6683a25", - "zh:74c89de3fa6ef3027efe08f8473c2baeb41b4c6cee250ba7aeb5b64e8c79800d", + "h1:nLWniS8xhb32qRQy+n4bDPjQ7YWZPVMR3v1vSrx7QyY=", + "zh:26e07aa32e403303fc212a4367b4d67188ac965c37a9812e07acee1470687a73", + "zh:27386f48e9c9d849fbb5a8828d461fde35e71f6b6c9fc235bc4ae8403eb9c92d", + "zh:5f4edda4c94240297bbd9b83618fd362348cadf6bf24ea65ea0e1844d7ccedc0", + "zh:646313a907126cd5e69f6a9fafe816e9154fccdc04541e06fed02bb3a8fa2d2e", + "zh:7349692932a5d462f8dee1500ab60401594dddb94e9aa6bf6c4c0bd53e91bbb8", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:b29eabbf0a5298f0e95a1df214c7cfe06ea9bcf362c63b3ad2f72d85da7d4685", - "zh:e891458c7a61e5b964e09616f1a4f87d0471feae1ec04cc51776e7dec1a3abce", + "zh:9034daba8d9b32b35930d168f363af04cecb153d5849a7e4a5966c97c5dc956e", + "zh:bb81dfca59ef5f949ef39f19ea4f4de25479907abc28cdaa36d12ecd7c0a9699", + "zh:bcf7806b99b4c248439ae02c8e21f77aff9fadbc019ce619b929eef09d1221bb", + "zh:d708e14d169e61f326535dd08eecd3811cd4942555a6f8efabc37dbff9c6fc61", + "zh:dc294e19a46e1cefb9e557a7b789c8dd8f319beca99b8c265181bc633dc434cc", + "zh:f9d758ee53c55dc016dd736427b6b0c3c8eb4d0dbbc785b6a3579b0ffedd9e42", ] } diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index b720640dc2..50bf276355 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -77,11 +77,11 @@ resource "azurerm_user_assigned_identity" "nexus_msi" { lifecycle { ignore_changes = [tags] } } -resource "azurerm_role_assignment" "kv_reader" { - scope = data.azurerm_key_vault.kv.id - role_definition_name = "Key Vault Reader" - principal_id = azurerm_user_assigned_identity.nexus_msi.principal_id -} +# resource "azurerm_role_assignment" "kv_reader" { +# scope = data.azurerm_key_vault.kv.id +# role_definition_name = "Key Vault Reader" +# principal_id = azurerm_user_assigned_identity.nexus_msi.principal_id +# } resource "azurerm_key_vault_access_policy" "nexus_msi" { key_vault_id = data.azurerm_key_vault.kv.id From 55b4864f99097f0f140b8b27eb13be9b7144f9b2 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 27 Apr 2022 11:06:12 +0000 Subject: [PATCH 075/142] Removed kv role assignment --- .../terraform/resource_processor/vmss_porter/main.tf | 2 +- .../sonatype-nexus/terraform/cloud-config.yaml | 2 +- templates/shared_services/sonatype-nexus/terraform/vm.tf | 9 +-------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index cc1067a790..7629fa5189 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -170,6 +170,6 @@ resource "azurerm_key_vault_access_policy" "resource_processor" { tenant_id = azurerm_user_assigned_identity.vmss_msi.tenant_id object_id = azurerm_user_assigned_identity.vmss_msi.principal_id - secret_permissions = ["Get", "List", "Set", "Delete"] + secret_permissions = ["Get", "List", "Set", "Delete", "Recover"] certificate_permissions = ["Get", "Update", "Create", "Import", "Delete"] } diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 37fd130f99..3d31e81c37 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -60,7 +60,7 @@ write_files: # set -o xtrace # Prepare ssl certificate - az login --identity -u ${msi_id} + az login --identity -u ${msi_id} --allow-no-subscriptions # -- get cert from kv as secret so it contains private key echo 'Getting cert and cert password from Keyvault...' az keyvault secret download --vault-name ${vault_name} --name ${ssl_cert_name} --file temp.pfx --encoding base64 diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 50bf276355..61b3e80c6b 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -77,18 +77,12 @@ resource "azurerm_user_assigned_identity" "nexus_msi" { lifecycle { ignore_changes = [tags] } } -# resource "azurerm_role_assignment" "kv_reader" { -# scope = data.azurerm_key_vault.kv.id -# role_definition_name = "Key Vault Reader" -# principal_id = azurerm_user_assigned_identity.nexus_msi.principal_id -# } - resource "azurerm_key_vault_access_policy" "nexus_msi" { key_vault_id = data.azurerm_key_vault.kv.id tenant_id = azurerm_user_assigned_identity.nexus_msi.tenant_id object_id = azurerm_user_assigned_identity.nexus_msi.principal_id - secret_permissions = ["Get"] + secret_permissions = ["Get", "Recover"] } resource "azurerm_linux_virtual_machine" "nexus" { @@ -128,7 +122,6 @@ resource "azurerm_linux_virtual_machine" "nexus" { } depends_on = [ - azurerm_role_assignment.kv_reader, azurerm_key_vault_access_policy.nexus_msi, azurerm_firewall_application_rule_collection.shared_subnet_nexus ] From a9609d13d50b2e209efbf260c6233f2aa3a345f2 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 27 Apr 2022 12:19:39 +0000 Subject: [PATCH 076/142] Adding firewall rule to allow letsencrypt from RP --- .../nexus-cert/terraform/data.tf | 11 ++++++++ .../nexus-cert/terraform/firewall.tf | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 templates/shared_services/nexus-cert/terraform/firewall.tf diff --git a/templates/shared_services/nexus-cert/terraform/data.tf b/templates/shared_services/nexus-cert/terraform/data.tf index 2d089e2019..fe79840003 100644 --- a/templates/shared_services/nexus-cert/terraform/data.tf +++ b/templates/shared_services/nexus-cert/terraform/data.tf @@ -18,3 +18,14 @@ data "azurerm_public_ip" "appgwpip_data" { name = "pip-nexus-${var.tre_id}" resource_group_name = data.azurerm_resource_group.rg.name } + +data "azurerm_subnet" "resource_processor" { + name = "ResourceProcessorSubnet" + virtual_network_name = "vnet-${var.tre_id}" + resource_group_name = local.core_resource_group_name +} + +data "azurerm_firewall" "fw" { + name = "fw-${var.tre_id}" + resource_group_name = local.core_resource_group_name +} diff --git a/templates/shared_services/nexus-cert/terraform/firewall.tf b/templates/shared_services/nexus-cert/terraform/firewall.tf new file mode 100644 index 0000000000..39b71ba036 --- /dev/null +++ b/templates/shared_services/nexus-cert/terraform/firewall.tf @@ -0,0 +1,25 @@ +resource "azurerm_firewall_application_rule_collection" "resource_processor_letsencrypt" { + name = "resource_processor_subnet_letsencrypt" + azure_firewall_name = data.azurerm_firewall.fw.name + resource_group_name = data.azurerm_firewall.fw.resource_group_name + priority = 106 + action = "Allow" + + rule { + name = "letsencrypt-acme" + protocol { + port = "443" + type = "Https" + } + protocol { + port = "80" + type = "Http" + } + + target_fqdns = [ + "https://acme-v02.api.letsencrypt.org/" + ] + + source_addresses = data.azurerm_subnet.resource_processor.address_prefixes + } +} From 8863ddf0678e0e18b3c6d5fe9b777990bca8c89a Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 27 Apr 2022 15:54:30 +0000 Subject: [PATCH 077/142] Genericised cert service and added letsencrypt action --- .../{nexus-cert => certs}/.dockerignore | 0 .../{nexus-cert => certs}/.gitignore | 0 .../{nexus-cert => certs}/Dockerfile.tmpl | 0 .../{nexus-cert => certs}/parameters.json | 0 templates/shared_services/certs/porter.yaml | 122 ++++++++++++++++++ .../scripts/auth-hook.sh | 0 .../scripts/cleanup-hook.sh | 0 .../scripts/json-to-env.sh | 0 .../scripts/letsencrypt.sh | 83 +++++++++--- .../{nexus-cert => certs}/scripts/outputs.sh | 0 .../certs/template_schema.json | 22 ++++ .../terraform/appgateway.tf | 10 +- .../terraform/certificate.tf | 2 +- .../{nexus-cert => certs}/terraform/data.tf | 0 .../terraform/firewall.tf | 0 .../shared_services/certs/terraform/locals.tf | 21 +++ .../{nexus-cert => certs}/terraform/main.tf | 0 .../{nexus-cert => certs}/terraform/output.tf | 4 + .../terraform/staticweb.tf | 0 .../certs/terraform/variables.tf | 3 + .../shared_services/nexus-cert/porter.yaml | 79 ------------ .../nexus-cert/template_schema.json | 9 -- .../nexus-cert/terraform/locals.tf | 23 ---- .../nexus-cert/terraform/variables.tf | 1 - 24 files changed, 241 insertions(+), 138 deletions(-) rename templates/shared_services/{nexus-cert => certs}/.dockerignore (100%) rename templates/shared_services/{nexus-cert => certs}/.gitignore (100%) rename templates/shared_services/{nexus-cert => certs}/Dockerfile.tmpl (100%) rename templates/shared_services/{nexus-cert => certs}/parameters.json (100%) create mode 100755 templates/shared_services/certs/porter.yaml rename templates/shared_services/{nexus-cert => certs}/scripts/auth-hook.sh (100%) rename templates/shared_services/{nexus-cert => certs}/scripts/cleanup-hook.sh (100%) rename templates/shared_services/{nexus-cert => certs}/scripts/json-to-env.sh (100%) rename templates/shared_services/{nexus-cert => certs}/scripts/letsencrypt.sh (61%) rename templates/shared_services/{nexus-cert => certs}/scripts/outputs.sh (100%) create mode 100644 templates/shared_services/certs/template_schema.json rename templates/shared_services/{nexus-cert => certs}/terraform/appgateway.tf (94%) rename templates/shared_services/{nexus-cert => certs}/terraform/certificate.tf (99%) rename templates/shared_services/{nexus-cert => certs}/terraform/data.tf (100%) rename templates/shared_services/{nexus-cert => certs}/terraform/firewall.tf (100%) create mode 100644 templates/shared_services/certs/terraform/locals.tf rename templates/shared_services/{nexus-cert => certs}/terraform/main.tf (100%) rename templates/shared_services/{nexus-cert => certs}/terraform/output.tf (87%) rename templates/shared_services/{nexus-cert => certs}/terraform/staticweb.tf (100%) create mode 100644 templates/shared_services/certs/terraform/variables.tf delete mode 100755 templates/shared_services/nexus-cert/porter.yaml delete mode 100644 templates/shared_services/nexus-cert/template_schema.json delete mode 100644 templates/shared_services/nexus-cert/terraform/locals.tf delete mode 100644 templates/shared_services/nexus-cert/terraform/variables.tf diff --git a/templates/shared_services/nexus-cert/.dockerignore b/templates/shared_services/certs/.dockerignore similarity index 100% rename from templates/shared_services/nexus-cert/.dockerignore rename to templates/shared_services/certs/.dockerignore diff --git a/templates/shared_services/nexus-cert/.gitignore b/templates/shared_services/certs/.gitignore similarity index 100% rename from templates/shared_services/nexus-cert/.gitignore rename to templates/shared_services/certs/.gitignore diff --git a/templates/shared_services/nexus-cert/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl similarity index 100% rename from templates/shared_services/nexus-cert/Dockerfile.tmpl rename to templates/shared_services/certs/Dockerfile.tmpl diff --git a/templates/shared_services/nexus-cert/parameters.json b/templates/shared_services/certs/parameters.json similarity index 100% rename from templates/shared_services/nexus-cert/parameters.json rename to templates/shared_services/certs/parameters.json diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml new file mode 100755 index 0000000000..2a4a58d721 --- /dev/null +++ b/templates/shared_services/certs/porter.yaml @@ -0,0 +1,122 @@ +--- +name: tre-shared-service-certs +version: 0.0.9 +description: "An Azure TRE shared service to generate certificates for a specified internal domain using Letsencrypt" +registry: azuretre +dockerfile: Dockerfile.tmpl + +credentials: + - name: azure_tenant_id + env: ARM_TENANT_ID + - name: azure_subscription_id + env: ARM_SUBSCRIPTION_ID + - name: azure_client_id + env: ARM_CLIENT_ID + - name: azure_client_secret + env: ARM_CLIENT_SECRET + +parameters: + - name: tre_id + type: string + description: "The ID of the parent TRE instance e.g., mytre-dev-3142" + - name: tfstate_resource_group_name + type: string + description: "Resource group containing the Terraform state storage account" + - name: tfstate_storage_account_name + type: string + description: "The name of the Terraform state storage account" + - name: tfstate_container_name + type: string + default: "tfstate" + description: "The name of the Terraform state storage container" + - name: arm_use_msi + env: ARM_USE_MSI + type: boolean + default: false + - name: domain_prefix + type: string + description: "The FQDN prefix (which will be prepended to {TRE_ID}.{LOCATION}.cloudapp.azure.com) to generate a certificate for" + - name: cert_name + type: string + description: "What to call the certificate exported to KeyVault" + +mixins: + - exec + - terraform: + clientVersion: 1.1.5 + +install: + - terraform: + description: "Deploy shared service" + input: false + vars: + tre_id: "{{ bundle.parameters.tre_id }}" + domain_prefix: "{{ bundle.parameters.domain_prefix }}" + cert_name: "{{ bundle.parameters.cert_name }}" + backendConfig: + resource_group_name: + "{{ bundle.parameters.tfstate_resource_group_name }}" + storage_account_name: + "{{ bundle.parameters.tfstate_storage_account_name }}" + container_name: + "{{ bundle.parameters.tfstate_container_name }}" + key: + "{{ bundle.parameters.tre_id }}-shared-service-certs" + +upgrade: + - exec: + description: "Upgrade shared service" + command: echo + arguments: + - "This shared service does not implement upgrade action" + +uninstall: + - terraform: + description: "Tear down shared service" + input: false + vars: + tre_id: "{{ bundle.parameters.tre_id }}" + domain_prefix: "{{ bundle.parameters.domain_prefix }}" + cert_name: "{{ bundle.parameters.cert_name }}" + backendConfig: + resource_group_name: + "{{ bundle.parameters.tfstate_resource_group_name }}" + storage_account_name: + "{{ bundle.parameters.tfstate_storage_account_name }}" + container_name: + "{{ bundle.parameters.tfstate_container_name }}" + key: + "{{ bundle.parameters.tre_id }}-shared-service-certs" + +generate: + - terraform: + arguments: + - "output" + description: "Get Terraform output variables" + backendConfig: + resource_group_name: + "{{ bundle.parameters.tfstate_resource_group_name }}" + storage_account_name: + "{{ bundle.parameters.tfstate_storage_account_name }}" + container_name: "{{ bundle.parameters.tfstate_container_name }}" + key: "{{ bundle.parameters.tre_id }}-shared-service-certs" + outputs: + - name: fqdn + - name: application_gateway_name + - name: storage_account_name + - name: storage_account_id + - name: resource_group_name + - name: keyvault_name + - exec: + description: "Generate/renew certificate" + command: bash + arguments: + - ./scripts/letsencrypt.sh + flags: + fqdn: "{{ bundle.outputs.fqdn }}" + application_gateway_name: "{{ bundle.outputs.application_gateway_name }}" + storage_account_name: "{{ bundle.outputs.storage_account_name }}" + storage_account_id: "{{ bundle.outputs.storage_account_id }}" + resource_group_name: "{{ bundle.outputs.resource_group_name }}" + keyvault_name: "{{ bundle.outputs.keyvault_name }}" + cert_name: "{{ bundle.parameters.cert_name }}" diff --git a/templates/shared_services/nexus-cert/scripts/auth-hook.sh b/templates/shared_services/certs/scripts/auth-hook.sh similarity index 100% rename from templates/shared_services/nexus-cert/scripts/auth-hook.sh rename to templates/shared_services/certs/scripts/auth-hook.sh diff --git a/templates/shared_services/nexus-cert/scripts/cleanup-hook.sh b/templates/shared_services/certs/scripts/cleanup-hook.sh similarity index 100% rename from templates/shared_services/nexus-cert/scripts/cleanup-hook.sh rename to templates/shared_services/certs/scripts/cleanup-hook.sh diff --git a/templates/shared_services/nexus-cert/scripts/json-to-env.sh b/templates/shared_services/certs/scripts/json-to-env.sh similarity index 100% rename from templates/shared_services/nexus-cert/scripts/json-to-env.sh rename to templates/shared_services/certs/scripts/json-to-env.sh diff --git a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh b/templates/shared_services/certs/scripts/letsencrypt.sh similarity index 61% rename from templates/shared_services/nexus-cert/scripts/letsencrypt.sh rename to templates/shared_services/certs/scripts/letsencrypt.sh index 09b550a5c2..9c515579ee 100755 --- a/templates/shared_services/nexus-cert/scripts/letsencrypt.sh +++ b/templates/shared_services/certs/scripts/letsencrypt.sh @@ -2,10 +2,53 @@ set -e script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") -if [[ -z ${STORAGE_ACCOUNT_NAME} ]]; then - echo "STORAGE_ACCOUNT not set" - exit 1 -fi + +while [ "$1" != "" ]; do + case $1 in + --storage_account_name) + shift + storage_account_name=$1 + ;; + --storage_account_id) + shift + storage_account_id=$1 + ;; + --fqdn) + shift + fqdn=$1 + ;; + --keyvault_name) + shift + keyvault_name=$1 + ;; + --resource_group_name) + shift + resource_group_name=$1 + ;; + --application_gateway_name) + shift + application_gateway_name=$1 + ;; + --cert_name) + shift + cert_name=$1 + ;; + *) + echo "Unexpected argument: '$1'" + usage + ;; + esac + + if [[ -z "$2" ]]; then + # if no more args then stop processing + break + fi + + shift # remove the current value for `$1` and use the next +done + +# done with processing args and can set this +set -o nounset echo "Checking for index.html file in storage account" @@ -13,7 +56,7 @@ echo "Checking for index.html file in storage account" objectId=$(az ad signed-in-user show --query objectId -o tsv) az role assignment create --assignee "${objectId}" \ --role "Storage Blob Data Contributor" \ - --scope "${STORAGE_ACCOUNT_ID}" + --scope "${storage_account_id}" # Create the default index.html page cat << EOF > index.html @@ -23,7 +66,7 @@ EOF # shellcheck disable=SC2016 indexExists=$(az storage blob list -o json \ - --account-name "${STORAGE_ACCOUNT_NAME}" \ + --account-name "${storage_account_name}" \ --auth-mode login \ --container-name '$web' \ --query "[?name=='index.html'].name" \ @@ -34,7 +77,7 @@ if [[ ${indexExists} -lt 1 ]]; then # shellcheck disable=SC2016 az storage blob upload \ - --account-name "${STORAGE_ACCOUNT_NAME}" \ + --account-name "${storage_account_name}" \ --auth-mode login \ --container-name '$web' \ --file index.html \ @@ -61,14 +104,14 @@ mkdir -p "${ledir}/logs" --preferred-challenges=http \ --manual-auth-hook "${script_dir}"/auth-hook.sh \ --manual-cleanup-hook "${script_dir}"/cleanup-hook.sh \ - --domain "${FQDN}" \ + --domain "${fqdn}" \ --non-interactive \ --agree-tos \ --register-unsafely-without-email # Convert the generated certificate to a .pfx -CERT_DIR="${ledir}/live/${FQDN}" +CERT_DIR="${ledir}/live/${fqdn}" CERT_PASSWORD=$(openssl rand -base64 30) openssl pkcs12 -export \ -inkey "${CERT_DIR}/privkey.pem" \ @@ -76,30 +119,30 @@ openssl pkcs12 -export \ -out "${CERT_DIR}/aci.pfx" \ -passout "pass:${CERT_PASSWORD}" -if [[ -n ${KEYVAULT} ]]; then +if [[ -n ${keyvault_name} ]]; then sid=$(az keyvault certificate import \ -o json \ - --vault-name "${KEYVAULT}" \ - --name 'nexus-letsencrypt' \ + --vault-name "${keyvault_name}" \ + --name "${cert_name}" \ --file "${CERT_DIR}/aci.pfx" \ --password "${CERT_PASSWORD}" \ | jq -r '.sid') # Save the certificate password to KV - az keyvault secret set --name nexus-letsencrypt-cert-password \ - --vault-name "${KEYVAULT}" \ + az keyvault secret set --name "${cert_name}"-password \ + --vault-name "${keyvault_name}" \ --value "${CERT_PASSWORD}" az network application-gateway ssl-cert update \ - --resource-group "${RESOURCE_GROUP_NAME}" \ - --gateway-name "${APPLICATION_GATEWAY}" \ - --name 'cert-nexus-primary' \ + --resource-group "${resource_group_name}" \ + --gateway-name "${application_gateway_name}" \ + --name 'cert-primary' \ --key-vault-secret-id "${sid}" else az network application-gateway ssl-cert update \ - --resource-group "${RESOURCE_GROUP_NAME}" \ - --gateway-name "${APPLICATION_GATEWAY}" \ - --name 'nexus-letsencrypt' \ + --resource-group "${resource_group_name}" \ + --gateway-name "${application_gateway_name}" \ + --name "${cert_name}" \ --cert-file "${CERT_DIR}/aci.pfx" \ --cert-password "${CERT_PASSWORD}" fi diff --git a/templates/shared_services/nexus-cert/scripts/outputs.sh b/templates/shared_services/certs/scripts/outputs.sh similarity index 100% rename from templates/shared_services/nexus-cert/scripts/outputs.sh rename to templates/shared_services/certs/scripts/outputs.sh diff --git a/templates/shared_services/certs/template_schema.json b/templates/shared_services/certs/template_schema.json new file mode 100644 index 0000000000..0f6ffacb40 --- /dev/null +++ b/templates/shared_services/certs/template_schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://github.com/microsoft/AzureTRE/templates/shared_services/sonatype-nexus/template_schema.json", + "type": "object", + "title": "Certs Service", + "description": "Provides SSL Certs for a specified internal domain", + "required": [], + "properties": { + "domain_prefix": { + "$id": "#/properties/domain_prefix", + "type": "string", + "title": "Domain prefix", + "description": "The FQDN prefix (which will be prepended to {TRE_ID}.{LOCATION}.cloudapp.azure.com) to generate a certificate for" + }, + "cert_name": { + "$id": "#/properties/cert_name", + "type": "string", + "title": "Cert name", + "description": "What to call the certificate that's exported to KeyVault" + } + } +} diff --git a/templates/shared_services/nexus-cert/terraform/appgateway.tf b/templates/shared_services/certs/terraform/appgateway.tf similarity index 94% rename from templates/shared_services/nexus-cert/terraform/appgateway.tf rename to templates/shared_services/certs/terraform/appgateway.tf index a0be40a584..6615676665 100644 --- a/templates/shared_services/nexus-cert/terraform/appgateway.tf +++ b/templates/shared_services/certs/terraform/appgateway.tf @@ -1,10 +1,10 @@ resource "azurerm_public_ip" "appgwpip" { - name = "pip-nexus-${var.tre_id}" + name = "pip-cert-${var.domain_prefix}-${var.tre_id}" resource_group_name = data.azurerm_resource_group.rg.name location = data.azurerm_resource_group.rg.location allocation_method = "Static" sku = "Standard" - domain_name_label = "nexus-${var.tre_id}" + domain_name_label = "${var.domain_prefix}-${var.tre_id}" lifecycle { ignore_changes = [tags] } } @@ -12,13 +12,13 @@ resource "azurerm_public_ip" "appgwpip" { resource "azurerm_user_assigned_identity" "agw_id" { resource_group_name = data.azurerm_resource_group.rg.name location = data.azurerm_resource_group.rg.location - name = "id-agw-nexuscert-${var.tre_id}" + name = "id-agw-certs-${var.tre_id}" lifecycle { ignore_changes = [tags] } } resource "azurerm_application_gateway" "agw" { - name = "agw-nexuscert-${var.tre_id}" + name = "agw-certs-${var.tre_id}" resource_group_name = data.azurerm_resource_group.rg.name location = data.azurerm_resource_group.rg.location @@ -60,7 +60,7 @@ resource "azurerm_application_gateway" "agw" { # Primary SSL cert linked to KeyVault ssl_certificate { - name = local.certificate_name + name = "cert-primary" key_vault_secret_id = azurerm_key_vault_certificate.tlscert.secret_id } diff --git a/templates/shared_services/nexus-cert/terraform/certificate.tf b/templates/shared_services/certs/terraform/certificate.tf similarity index 99% rename from templates/shared_services/nexus-cert/terraform/certificate.tf rename to templates/shared_services/certs/terraform/certificate.tf index 938354c730..44d39980c2 100644 --- a/templates/shared_services/nexus-cert/terraform/certificate.tf +++ b/templates/shared_services/certs/terraform/certificate.tf @@ -8,7 +8,7 @@ resource "azurerm_key_vault_access_policy" "app_gw_managed_identity" { } resource "azurerm_key_vault_certificate" "tlscert" { - name = "nexus-letsencrypt" + name = var.cert_name key_vault_id = data.azurerm_key_vault.key_vault.id # This is a temporary self-signed cert for CN=temp diff --git a/templates/shared_services/nexus-cert/terraform/data.tf b/templates/shared_services/certs/terraform/data.tf similarity index 100% rename from templates/shared_services/nexus-cert/terraform/data.tf rename to templates/shared_services/certs/terraform/data.tf diff --git a/templates/shared_services/nexus-cert/terraform/firewall.tf b/templates/shared_services/certs/terraform/firewall.tf similarity index 100% rename from templates/shared_services/nexus-cert/terraform/firewall.tf rename to templates/shared_services/certs/terraform/firewall.tf diff --git a/templates/shared_services/certs/terraform/locals.tf b/templates/shared_services/certs/terraform/locals.tf new file mode 100644 index 0000000000..2666d7aa8b --- /dev/null +++ b/templates/shared_services/certs/terraform/locals.tf @@ -0,0 +1,21 @@ +locals { + staticweb_storage_name = lower(replace("stwebnexus${var.tre_id}", "-", "")) + + staticweb_backend_pool_name = "beap-certs-staticweb" + app_path_map_name = "upm-certs" + redirect_path_map_name = "upm-certs-redirect" + + insecure_frontend_port_name = "feport-certs-insecure" + secure_frontend_port_name = "feport-certs-secure" + + frontend_ip_configuration_name = "feip-certs-public" + + staticweb_http_setting_name = "be-htst-certs-staticweb" + + insecure_listener_name = "httplstn-certs-insecure" + secure_listener_name = "httplstn-certs-secure" + + redirect_request_routing_rule_name = "rqrt-certs-redirect" + request_routing_rule_name = "rqrt-certs-application" + redirect_configuration_name = "rdrcfg-certs-tosecure" +} diff --git a/templates/shared_services/nexus-cert/terraform/main.tf b/templates/shared_services/certs/terraform/main.tf similarity index 100% rename from templates/shared_services/nexus-cert/terraform/main.tf rename to templates/shared_services/certs/terraform/main.tf diff --git a/templates/shared_services/nexus-cert/terraform/output.tf b/templates/shared_services/certs/terraform/output.tf similarity index 87% rename from templates/shared_services/nexus-cert/terraform/output.tf rename to templates/shared_services/certs/terraform/output.tf index d8a4b8d2b3..528b48cab7 100644 --- a/templates/shared_services/nexus-cert/terraform/output.tf +++ b/templates/shared_services/certs/terraform/output.tf @@ -21,3 +21,7 @@ output "resource_group_name" { output "keyvault" { value = data.azurerm_key_vault.key_vault.name } + +output "temp_certificate_name" { + value = locals.certificate_name +} diff --git a/templates/shared_services/nexus-cert/terraform/staticweb.tf b/templates/shared_services/certs/terraform/staticweb.tf similarity index 100% rename from templates/shared_services/nexus-cert/terraform/staticweb.tf rename to templates/shared_services/certs/terraform/staticweb.tf diff --git a/templates/shared_services/certs/terraform/variables.tf b/templates/shared_services/certs/terraform/variables.tf new file mode 100644 index 0000000000..29c51df8c3 --- /dev/null +++ b/templates/shared_services/certs/terraform/variables.tf @@ -0,0 +1,3 @@ +variable "tre_id" {} +variable "domain_prefix" {} +variable "cert_name" {} diff --git a/templates/shared_services/nexus-cert/porter.yaml b/templates/shared_services/nexus-cert/porter.yaml deleted file mode 100755 index 4160d241fe..0000000000 --- a/templates/shared_services/nexus-cert/porter.yaml +++ /dev/null @@ -1,79 +0,0 @@ ---- -name: tre-shared-service-nexus-cert -version: 0.0.9 -description: "An Azure TRE Nexus certificate creation shared service" -registry: azuretre -dockerfile: Dockerfile.tmpl - -credentials: - - name: azure_tenant_id - env: ARM_TENANT_ID - - name: azure_subscription_id - env: ARM_SUBSCRIPTION_ID - - name: azure_client_id - env: ARM_CLIENT_ID - - name: azure_client_secret - env: ARM_CLIENT_SECRET - -parameters: - - name: tre_id - type: string - description: "The ID of the parent TRE instance e.g., mytre-dev-3142" - - name: tfstate_resource_group_name - type: string - description: "Resource group containing the Terraform state storage account" - - name: tfstate_storage_account_name - type: string - description: "The name of the Terraform state storage account" - - name: tfstate_container_name - type: string - default: "tfstate" - description: "The name of the Terraform state storage container" - - name: arm_use_msi - env: ARM_USE_MSI - type: boolean - default: false - -mixins: - - exec - - terraform: - clientVersion: 1.1.5 - -install: - - terraform: - description: "Deploy shared service" - input: false - vars: - tre_id: "{{ bundle.parameters.tre_id }}" - backendConfig: - resource_group_name: - "{{ bundle.parameters.tfstate_resource_group_name }}" - storage_account_name: - "{{ bundle.parameters.tfstate_storage_account_name }}" - container_name: - "{{ bundle.parameters.tfstate_container_name }}" - key: - "{{ bundle.parameters.tre_id }}-shared-service-nexus-cert" - -upgrade: - - exec: - description: "Upgrade shared service" - command: echo - arguments: - - "This shared service does not implement upgrade action" - -uninstall: - - terraform: - description: "Tear down shared service" - input: false - vars: - tre_id: "{{ bundle.parameters.tre_id }}" - backendConfig: - resource_group_name: - "{{ bundle.parameters.tfstate_resource_group_name }}" - storage_account_name: - "{{ bundle.parameters.tfstate_storage_account_name }}" - container_name: - "{{ bundle.parameters.tfstate_container_name }}" - key: - "{{ bundle.parameters.tre_id }}-shared-service-nexus-cert" diff --git a/templates/shared_services/nexus-cert/template_schema.json b/templates/shared_services/nexus-cert/template_schema.json deleted file mode 100644 index c8e59956b8..0000000000 --- a/templates/shared_services/nexus-cert/template_schema.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://github.com/microsoft/AzureTRE/templates/shared_services/sonatype-nexus/template_schema.json", - "type": "object", - "title": "Nexus Cert Service", - "description": "Provides SSL Cert for Nexus shared service", - "required": [], - "properties": {} -} diff --git a/templates/shared_services/nexus-cert/terraform/locals.tf b/templates/shared_services/nexus-cert/terraform/locals.tf deleted file mode 100644 index 579394a4ff..0000000000 --- a/templates/shared_services/nexus-cert/terraform/locals.tf +++ /dev/null @@ -1,23 +0,0 @@ -locals { - staticweb_storage_name = lower(replace("stwebnexus${var.tre_id}", "-", "")) - - staticweb_backend_pool_name = "beap-nexuscret-staticweb" - app_path_map_name = "upm-nexuscert" - redirect_path_map_name = "upm-nexuscert-redirect" - - insecure_frontend_port_name = "feport-nexuscert-insecure" - secure_frontend_port_name = "feport-nexuscert-secure" - - frontend_ip_configuration_name = "feip-nexuscert-public" - - staticweb_http_setting_name = "be-htst-nexuscert-staticweb" - - insecure_listener_name = "httplstn-nexuscert-insecure" - secure_listener_name = "httplstn-nexuscert-secure" - - redirect_request_routing_rule_name = "rqrt-nexuscert-redirect" - request_routing_rule_name = "rqrt-nexuscert-application" - redirect_configuration_name = "rdrcfg-nexuscert-tosecure" - - certificate_name = "cert-nexus-primary" -} diff --git a/templates/shared_services/nexus-cert/terraform/variables.tf b/templates/shared_services/nexus-cert/terraform/variables.tf deleted file mode 100644 index 798db79a9f..0000000000 --- a/templates/shared_services/nexus-cert/terraform/variables.tf +++ /dev/null @@ -1 +0,0 @@ -variable "tre_id" {} From 74405ea6e3c38dd6ce178e97f80d4cd75d26e8b9 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 27 Apr 2022 16:04:56 +0000 Subject: [PATCH 078/142] Fixed auth hook --- .../certs/scripts/json-to-env.sh | 50 ------------------- .../certs/scripts/letsencrypt.sh | 1 + .../shared_services/certs/scripts/outputs.sh | 20 -------- .../shared_services/certs/terraform/locals.tf | 2 +- 4 files changed, 2 insertions(+), 71 deletions(-) delete mode 100755 templates/shared_services/certs/scripts/json-to-env.sh delete mode 100755 templates/shared_services/certs/scripts/outputs.sh diff --git a/templates/shared_services/certs/scripts/json-to-env.sh b/templates/shared_services/certs/scripts/json-to-env.sh deleted file mode 100755 index 5926e4edb1..0000000000 --- a/templates/shared_services/certs/scripts/json-to-env.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -set -e - -echo "# Generated environment variables from tf output" - -jq -r ' - [ - { - "path": "fqdn", - "env_var": "FQDN" - }, - { - "path": "application_gateway", - "env_var": "APPLICATION_GATEWAY" - }, - { - "path": "storage_account_id", - "env_var": "STORAGE_ACCOUNT_ID" - }, - { - "path": "storage_account_name", - "env_var": "STORAGE_ACCOUNT_NAME" - }, - { - "path": "resource_group_name", - "env_var": "RESOURCE_GROUP_NAME" - }, - { - "path": "keyvault", - "env_var": "KEYVAULT" - } - ] - as $env_vars_to_extract - | - with_entries( - select ( - .key as $a - | - any( $env_vars_to_extract[]; .path == $a) - ) - | - .key |= . as $old_key | ($env_vars_to_extract[] | select (.path == $old_key) | .env_var) - ) - | - to_entries - | - map("\(.key)=\"\(.value.value)\"") - | - .[] - ' | sed "s/\"/'/g" # replace double quote with single quote to handle special chars diff --git a/templates/shared_services/certs/scripts/letsencrypt.sh b/templates/shared_services/certs/scripts/letsencrypt.sh index 9c515579ee..e3b45aaa1e 100755 --- a/templates/shared_services/certs/scripts/letsencrypt.sh +++ b/templates/shared_services/certs/scripts/letsencrypt.sh @@ -96,6 +96,7 @@ ledir="${script_dir}/../letsencrypt" mkdir -p "${ledir}/logs" # Initiate the ACME challange +export STORAGE_ACCOUNT_NAME="${storage_account_name}" /opt/certbot/bin/certbot certonly \ --config-dir "${ledir}" \ --work-dir "${ledir}" \ diff --git a/templates/shared_services/certs/scripts/outputs.sh b/templates/shared_services/certs/scripts/outputs.sh deleted file mode 100755 index 753d63b672..0000000000 --- a/templates/shared_services/certs/scripts/outputs.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -e - -if [ ! -f ../tre_output.json ]; then - # Connect to the remote backend of Terraform - pushd ../terraform > /dev/null - export TF_LOG="" - terraform init -input=false -backend=true -reconfigure -upgrade \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name:?}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name:?}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name:?}" \ - -backend-config="key=${TRE_ID}-shared-service-nexus-cert" - - # Convert the output to json - terraform output -json > ../tre_output.json - popd > /dev/null -fi - -# Now create an .env file -./json-to-env.sh < ../tre_output.json > ../.env diff --git a/templates/shared_services/certs/terraform/locals.tf b/templates/shared_services/certs/terraform/locals.tf index 2666d7aa8b..3ea8558a13 100644 --- a/templates/shared_services/certs/terraform/locals.tf +++ b/templates/shared_services/certs/terraform/locals.tf @@ -1,5 +1,5 @@ locals { - staticweb_storage_name = lower(replace("stwebnexus${var.tre_id}", "-", "")) + staticweb_storage_name = lower(replace("stwebcerts${var.tre_id}", "-", "")) staticweb_backend_pool_name = "beap-certs-staticweb" app_path_map_name = "upm-certs" From b14bd19c8a74c5b7936e41e08baed47e1077b973 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 27 Apr 2022 16:08:02 +0000 Subject: [PATCH 079/142] Removed make commands --- Makefile | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/Makefile b/Makefile index 8777432ab9..5d05309479 100644 --- a/Makefile +++ b/Makefile @@ -106,12 +106,6 @@ gitea-install: && $(MAKE) bundle-register DIR="./templates/shared_services/gitea" BUNDLE_TYPE=shared_service \ && $(MAKE) deploy-shared-service DIR=./templates/shared_services/gitea/ BUNDLE_TYPE=shared_service -nexus-cert-install: - $(MAKE) bundle-build DIR=./templates/shared_services/nexus-cert/ \ - && $(MAKE) bundle-publish DIR=./templates/shared_services/nexus-cert/ \ - && $(MAKE) bundle-register DIR="./templates/shared_services/nexus-cert" BUNDLE_TYPE=shared_service \ - && $(MAKE) deploy-shared-service DIR=./templates/shared_services/nexus-cert/ BUNDLE_TYPE=shared_service - # A recipe for pushing images. Parameters: # 1. Image name suffix # 2. Version file path @@ -159,17 +153,6 @@ prepare-tf-state: # / End migration targets -nexus-letsencrypt: - $(call target_title, "Requesting LetsEncrypt SSL certificate for Nexus") \ - && . ./devops/scripts/check_dependencies.sh nodocker,certbot \ - && . ./devops/scripts/load_env.sh ./templates/core/.env \ - && . ./devops/scripts/load_env.sh ./devops/.env \ - && . ./devops/scripts/load_terraform_env.sh ./devops/.env \ - && . ./devops/scripts/load_terraform_env.sh ./templates/core/.env \ - && pushd ./templates/shared_services/nexus-cert/scripts/ > /dev/null && . ./outputs.sh && popd > /dev/null \ - && . ./devops/scripts/load_env.sh ./templates/shared_services/nexus-cert/.env \ - && ./templates/shared_services/nexus-cert/scripts/letsencrypt.sh - deploy-core: tre-start $(call target_title, "Deploying TRE") \ && . ./devops/scripts/check_dependencies.sh nodocker \ From b452f28fb4e1ba82f0bdd6cc2dba131a877dd093 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 27 Apr 2022 16:10:53 +0000 Subject: [PATCH 080/142] Certbot in bundle container --- .../shared_services/certs/Dockerfile.tmpl | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 0832adde5a..143968038f 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -15,18 +15,12 @@ RUN apt-get update \ && apt-get update && apt-get -y install azure-cli \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# This is a template Dockerfile for the bundle's invocation image -# You can customize it to use different base images, install tools and copy configuration files. -# -# Porter will use it as a template and append lines to it for the mixins -# and to set the CMD appropriately for the CNAB specification. -# -# Add the following line to porter.yaml to instruct Porter to use this template -# dockerfile: Dockerfile.tmpl - -# You can control where the mixin's Dockerfile lines are inserted into this file by moving "# PORTER_MIXINS" line -# another location in this file. If you remove that line, the mixins generated content is appended to this file. -# PORTER_MIXINS +# Install Certbot +RUN apt-get update && apt-get install -y python3 python3-venv libaugeas0 \ + && python3 -m venv /opt/certbot/ \ + && /opt/certbot/bin/pip install --no-cache-dir --upgrade pip \ + && /opt/certbot/bin/pip install --no-cache-dir certbot \ + && apt-get clean -y && rm -rf /var/lib/apt/lists/* # Use the BUNDLE_DIR build argument to copy files into the bundle COPY . $BUNDLE_DIR From b61ee25b2eb40331b2f135e12a03363253f4d097 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 27 Apr 2022 16:16:46 +0000 Subject: [PATCH 081/142] Tidied naming --- templates/shared_services/certs/terraform/output.tf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/templates/shared_services/certs/terraform/output.tf b/templates/shared_services/certs/terraform/output.tf index 528b48cab7..31b529097b 100644 --- a/templates/shared_services/certs/terraform/output.tf +++ b/templates/shared_services/certs/terraform/output.tf @@ -2,7 +2,7 @@ output "fqdn" { value = data.azurerm_public_ip.appgwpip_data.fqdn } -output "application_gateway" { +output "application_gateway_name" { value = azurerm_application_gateway.agw.name } @@ -18,10 +18,6 @@ output "resource_group_name" { value = azurerm_application_gateway.agw.resource_group_name } -output "keyvault" { +output "keyvault_name" { value = data.azurerm_key_vault.key_vault.name } - -output "temp_certificate_name" { - value = locals.certificate_name -} From c9d55685ef5877c19f0e32b01e1bae87ff7c43cb Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 27 Apr 2022 22:13:10 +0000 Subject: [PATCH 082/142] Python base image --- templates/shared_services/certs/Dockerfile.tmpl | 2 +- templates/shared_services/certs/porter.yaml | 2 +- templates/shared_services/certs/terraform/appgateway.tf | 2 +- templates/shared_services/certs/terraform/data.tf | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 143968038f..d7e1a1f563 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -1,4 +1,4 @@ -FROM debian:stretch-slim +FROM python:3.8 ARG BUNDLE_DIR diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index 2a4a58d721..2025016d0e 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-certs -version: 0.0.9 +version: 0.0.10 description: "An Azure TRE shared service to generate certificates for a specified internal domain using Letsencrypt" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/shared_services/certs/terraform/appgateway.tf b/templates/shared_services/certs/terraform/appgateway.tf index 6615676665..4259d73c47 100644 --- a/templates/shared_services/certs/terraform/appgateway.tf +++ b/templates/shared_services/certs/terraform/appgateway.tf @@ -86,7 +86,7 @@ resource "azurerm_application_gateway" "agw" { frontend_ip_configuration_name = local.frontend_ip_configuration_name frontend_port_name = local.secure_frontend_port_name protocol = "Https" - ssl_certificate_name = local.certificate_name + ssl_certificate_name = "cert-primary" } # Public HTTP listener diff --git a/templates/shared_services/certs/terraform/data.tf b/templates/shared_services/certs/terraform/data.tf index fe79840003..f0bd1e73bd 100644 --- a/templates/shared_services/certs/terraform/data.tf +++ b/templates/shared_services/certs/terraform/data.tf @@ -22,10 +22,10 @@ data "azurerm_public_ip" "appgwpip_data" { data "azurerm_subnet" "resource_processor" { name = "ResourceProcessorSubnet" virtual_network_name = "vnet-${var.tre_id}" - resource_group_name = local.core_resource_group_name + resource_group_name = data.azurerm_resource_group.rg.name } data "azurerm_firewall" "fw" { name = "fw-${var.tre_id}" - resource_group_name = local.core_resource_group_name + resource_group_name = data.azurerm_resource_group.rg.name } From 06b6c40720d0ae3b310cb141603cf1a521603eac Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 28 Apr 2022 13:47:25 +0000 Subject: [PATCH 083/142] Generate action successful --- templates/shared_services/certs/porter.yaml | 13 +++- .../certs/scripts/letsencrypt.sh | 64 ++++++++----------- .../certs/template_schema.json | 2 +- .../shared_services/certs/terraform/data.tf | 11 ++-- .../certs/terraform/firewall.tf | 2 +- .../shared_services/certs/terraform/output.tf | 6 +- .../certs/terraform/staticweb.tf | 6 ++ .../certs/terraform/variables.tf | 14 +++- 8 files changed, 60 insertions(+), 58 deletions(-) diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index 2025016d0e..ef1a8dd0f8 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -38,12 +38,13 @@ parameters: description: "The FQDN prefix (which will be prepended to {TRE_ID}.{LOCATION}.cloudapp.azure.com) to generate a certificate for" - name: cert_name type: string - description: "What to call the certificate exported to KeyVault" + description: "What to call the certificate exported to KeyVault (alphanumeric and '-' only)" mixins: - exec - terraform: clientVersion: 1.1.5 + - az install: - terraform: @@ -104,9 +105,16 @@ generate: - name: fqdn - name: application_gateway_name - name: storage_account_name - - name: storage_account_id - name: resource_group_name - name: keyvault_name + - az: + description: + "Login to Azure" + arguments: + - login + flags: + identity: + username: "{{ bundle.credentials.azure_client_id }}" - exec: description: "Generate/renew certificate" command: bash @@ -116,7 +124,6 @@ generate: fqdn: "{{ bundle.outputs.fqdn }}" application_gateway_name: "{{ bundle.outputs.application_gateway_name }}" storage_account_name: "{{ bundle.outputs.storage_account_name }}" - storage_account_id: "{{ bundle.outputs.storage_account_id }}" resource_group_name: "{{ bundle.outputs.resource_group_name }}" keyvault_name: "{{ bundle.outputs.keyvault_name }}" cert_name: "{{ bundle.parameters.cert_name }}" diff --git a/templates/shared_services/certs/scripts/letsencrypt.sh b/templates/shared_services/certs/scripts/letsencrypt.sh index e3b45aaa1e..4339990f43 100755 --- a/templates/shared_services/certs/scripts/letsencrypt.sh +++ b/templates/shared_services/certs/scripts/letsencrypt.sh @@ -9,10 +9,6 @@ while [ "$1" != "" ]; do shift storage_account_name=$1 ;; - --storage_account_id) - shift - storage_account_id=$1 - ;; --fqdn) shift fqdn=$1 @@ -52,12 +48,6 @@ set -o nounset echo "Checking for index.html file in storage account" -# Assign Storage Blob Data Contributor permissions if not already present -objectId=$(az ad signed-in-user show --query objectId -o tsv) -az role assignment create --assignee "${objectId}" \ - --role "Storage Blob Data Contributor" \ - --scope "${storage_account_id}" - # Create the default index.html page cat << EOF > index.html @@ -73,7 +63,7 @@ indexExists=$(az storage blob list -o json \ | jq 'length') if [[ ${indexExists} -lt 1 ]]; then - echo "Uploading index.html file" + echo "No existing file found. Uploading index.html file" # shellcheck disable=SC2016 az storage blob upload \ @@ -86,7 +76,7 @@ if [[ ${indexExists} -lt 1 ]]; then --only-show-errors # Wait a bit for the App Gateway health probe to notice - echo "Waiting 30s for health probe" + echo "Waiting 30s for app gateway health probe" sleep 30s else echo "index.html already present" @@ -96,6 +86,7 @@ ledir="${script_dir}/../letsencrypt" mkdir -p "${ledir}/logs" # Initiate the ACME challange +echo "Initiating ACME challenge" export STORAGE_ACCOUNT_NAME="${storage_account_name}" /opt/certbot/bin/certbot certonly \ --config-dir "${ledir}" \ @@ -112,6 +103,7 @@ export STORAGE_ACCOUNT_NAME="${storage_account_name}" # Convert the generated certificate to a .pfx +echo "Got cert. Converting to PFX" CERT_DIR="${ledir}/live/${fqdn}" CERT_PASSWORD=$(openssl rand -base64 30) openssl pkcs12 -export \ @@ -120,30 +112,24 @@ openssl pkcs12 -export \ -out "${CERT_DIR}/aci.pfx" \ -passout "pass:${CERT_PASSWORD}" -if [[ -n ${keyvault_name} ]]; then - sid=$(az keyvault certificate import \ - -o json \ - --vault-name "${keyvault_name}" \ - --name "${cert_name}" \ - --file "${CERT_DIR}/aci.pfx" \ - --password "${CERT_PASSWORD}" \ - | jq -r '.sid') - - # Save the certificate password to KV - az keyvault secret set --name "${cert_name}"-password \ - --vault-name "${keyvault_name}" \ - --value "${CERT_PASSWORD}" - - az network application-gateway ssl-cert update \ - --resource-group "${resource_group_name}" \ - --gateway-name "${application_gateway_name}" \ - --name 'cert-primary' \ - --key-vault-secret-id "${sid}" -else - az network application-gateway ssl-cert update \ - --resource-group "${resource_group_name}" \ - --gateway-name "${application_gateway_name}" \ - --name "${cert_name}" \ - --cert-file "${CERT_DIR}/aci.pfx" \ - --cert-password "${CERT_PASSWORD}" -fi +# Save cert and password to KeyVault +echo "Importing cert to KeyVault ${keyvault_name}" +sid=$(az keyvault certificate import \ + -o json \ + --vault-name "${keyvault_name}" \ + --name "${cert_name}" \ + --file "${CERT_DIR}/aci.pfx" \ + --password "${CERT_PASSWORD}" \ + | jq -r '.sid') + +echo "Saving certificate password to KV with key ${cert_name}-password" +az keyvault secret set --name "${cert_name}"-password \ + --vault-name "${keyvault_name}" \ + --value "${CERT_PASSWORD}" + +echo "Updating SSL cert in app gateway" +az network application-gateway ssl-cert update \ + --resource-group "${resource_group_name}" \ + --gateway-name "${application_gateway_name}" \ + --name 'cert-primary' \ + --key-vault-secret-id "${sid}" diff --git a/templates/shared_services/certs/template_schema.json b/templates/shared_services/certs/template_schema.json index 0f6ffacb40..b28e76bc5f 100644 --- a/templates/shared_services/certs/template_schema.json +++ b/templates/shared_services/certs/template_schema.json @@ -16,7 +16,7 @@ "$id": "#/properties/cert_name", "type": "string", "title": "Cert name", - "description": "What to call the certificate that's exported to KeyVault" + "description": "What to call the certificate that's exported to KeyVault (alphanumeric and '-' only)" } } } diff --git a/templates/shared_services/certs/terraform/data.tf b/templates/shared_services/certs/terraform/data.tf index f0bd1e73bd..47db962c89 100644 --- a/templates/shared_services/certs/terraform/data.tf +++ b/templates/shared_services/certs/terraform/data.tf @@ -13,12 +13,6 @@ data "azurerm_subnet" "app_gw_subnet" { resource_group_name = data.azurerm_resource_group.rg.name } -data "azurerm_public_ip" "appgwpip_data" { - depends_on = [azurerm_application_gateway.agw] - name = "pip-nexus-${var.tre_id}" - resource_group_name = data.azurerm_resource_group.rg.name -} - data "azurerm_subnet" "resource_processor" { name = "ResourceProcessorSubnet" virtual_network_name = "vnet-${var.tre_id}" @@ -29,3 +23,8 @@ data "azurerm_firewall" "fw" { name = "fw-${var.tre_id}" resource_group_name = data.azurerm_resource_group.rg.name } + +data "azurerm_user_assigned_identity" "resource_processor_vmss_id" { + name = "id-vmss-${var.tre_id}" + resource_group_name = "rg-${var.tre_id}" +} diff --git a/templates/shared_services/certs/terraform/firewall.tf b/templates/shared_services/certs/terraform/firewall.tf index 39b71ba036..d7b4b60930 100644 --- a/templates/shared_services/certs/terraform/firewall.tf +++ b/templates/shared_services/certs/terraform/firewall.tf @@ -17,7 +17,7 @@ resource "azurerm_firewall_application_rule_collection" "resource_processor_lets } target_fqdns = [ - "https://acme-v02.api.letsencrypt.org/" + "acme-v02.api.letsencrypt.org" ] source_addresses = data.azurerm_subnet.resource_processor.address_prefixes diff --git a/templates/shared_services/certs/terraform/output.tf b/templates/shared_services/certs/terraform/output.tf index 31b529097b..882e91b2da 100644 --- a/templates/shared_services/certs/terraform/output.tf +++ b/templates/shared_services/certs/terraform/output.tf @@ -1,5 +1,5 @@ output "fqdn" { - value = data.azurerm_public_ip.appgwpip_data.fqdn + value = azurerm_public_ip.appgwpip.fqdn } output "application_gateway_name" { @@ -10,10 +10,6 @@ output "storage_account_name" { value = azurerm_storage_account.staticweb.name } -output "storage_account_id" { - value = azurerm_storage_account.staticweb.id -} - output "resource_group_name" { value = azurerm_application_gateway.agw.resource_group_name } diff --git a/templates/shared_services/certs/terraform/staticweb.tf b/templates/shared_services/certs/terraform/staticweb.tf index d882900533..d6a315cd89 100644 --- a/templates/shared_services/certs/terraform/staticweb.tf +++ b/templates/shared_services/certs/terraform/staticweb.tf @@ -20,3 +20,9 @@ resource "azurerm_storage_account" "staticweb" { lifecycle { ignore_changes = [tags] } } + +resource "azurerm_role_assignment" "stgwriter" { + scope = azurerm_storage_account.staticweb.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_user_assigned_identity.resource_processor_vmss_id.principal_id +} diff --git a/templates/shared_services/certs/terraform/variables.tf b/templates/shared_services/certs/terraform/variables.tf index 29c51df8c3..cc3391ad14 100644 --- a/templates/shared_services/certs/terraform/variables.tf +++ b/templates/shared_services/certs/terraform/variables.tf @@ -1,3 +1,11 @@ -variable "tre_id" {} -variable "domain_prefix" {} -variable "cert_name" {} +variable "tre_id" { + type = string +} + +variable "domain_prefix" { + type = string +} + +variable "cert_name" { + type = string +} From 146ef52a23d76fa4a26880aaa1d7812157448cc7 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 28 Apr 2022 14:16:07 +0000 Subject: [PATCH 084/142] Inject cert name to nexus bundle --- templates/shared_services/sonatype-nexus/porter.yaml | 9 ++++++++- .../sonatype-nexus/template_schema.json | 9 ++++++++- .../shared_services/sonatype-nexus/terraform/data.tf | 2 +- .../sonatype-nexus/terraform/locals.tf | 3 ++- .../sonatype-nexus/terraform/variables.tf | 11 +---------- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/porter.yaml b/templates/shared_services/sonatype-nexus/porter.yaml index 150565da27..b42c264a2b 100644 --- a/templates/shared_services/sonatype-nexus/porter.yaml +++ b/templates/shared_services/sonatype-nexus/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-nexus -version: 0.0.2 +version: 0.0.3 description: "A Sonatype Nexus shared service" registry: azuretre credentials: @@ -29,6 +29,11 @@ parameters: - name: arm_use_msi env: ARM_USE_MSI default: false + - name: ssl_cert_name + type: string + default: "nexus-ssl" + description: "Name of the certificate for configuring Nexus SSL with (stored in the core KeyVault)" + mixins: - exec - az @@ -40,6 +45,7 @@ install: input: false vars: tre_id: "{{ bundle.parameters.tre_id }}" + ssl_cert_name: "{{ bundle.parameters.ssl_cert_name }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -59,6 +65,7 @@ uninstall: input: false vars: tre_id: "{{ bundle.parameters.tre_id }}" + ssl_cert_name: "{{ bundle.parameters.ssl_cert_name }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" diff --git a/templates/shared_services/sonatype-nexus/template_schema.json b/templates/shared_services/sonatype-nexus/template_schema.json index 13e66df1a7..dc0685c814 100644 --- a/templates/shared_services/sonatype-nexus/template_schema.json +++ b/templates/shared_services/sonatype-nexus/template_schema.json @@ -5,5 +5,12 @@ "title": "Nexus Shared Service", "description": "Provides Nexus shared service", "required": [], - "properties": {} + "properties": { + "ssl_cert_name": { + "type": "string", + "title": "SSL certificate name", + "description": "The name of the certificate to use (located in the core KeyVault) for configuring Nexus SSL", + "default": "nexus-ssl" + } + } } diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus/terraform/data.tf index 8318f88d69..a806a131ec 100644 --- a/templates/shared_services/sonatype-nexus/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus/terraform/data.tf @@ -25,7 +25,7 @@ data "azurerm_key_vault" "kv" { } data "azurerm_key_vault_certificate" "nexus_cert" { - name = "nexus-letsencrypt" + name = var.ssl_cert_name key_vault_id = data.azurerm_key_vault.kv.id } diff --git a/templates/shared_services/sonatype-nexus/terraform/locals.tf b/templates/shared_services/sonatype-nexus/terraform/locals.tf index 566f1c102c..d9736d661d 100644 --- a/templates/shared_services/sonatype-nexus/terraform/locals.tf +++ b/templates/shared_services/sonatype-nexus/terraform/locals.tf @@ -2,6 +2,7 @@ locals { core_vnet = "vnet-${var.tre_id}" core_resource_group_name = "rg-${var.tre_id}" firewall_name = "fw-${var.tre_id}" - nexus_allowed_fqdns_list = distinct(compact(split(",", replace(var.nexus_allowed_fqdns, " ", "")))) + nexus_allowed_fqdns = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com,keyserver.ubuntu.com,repo.anaconda.com,*.docker.com,*.docker.io,conda.anaconda.org,azure.archive.ubuntu.com, packages.microsoft.com" + nexus_allowed_fqdns_list = distinct(compact(split(",", replace(local.nexus_allowed_fqdns, " ", "")))) storage_account_name = lower(replace("stg-${var.tre_id}", "-", "")) } diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus/terraform/variables.tf index 0f5393d583..d63fda333f 100644 --- a/templates/shared_services/sonatype-nexus/terraform/variables.tf +++ b/templates/shared_services/sonatype-nexus/terraform/variables.tf @@ -1,16 +1,7 @@ variable "tre_id" { type = string - description = "Unique TRE ID" } -variable "nexus_storage_limit" { - type = number - description = "Space allocated in GB for the Nexus data in Azure Files Share" - default = 1024 -} - -variable "nexus_allowed_fqdns" { +variable "ssl_cert_name" { type = string - description = "comma seperated string of allowed FQDNs for Nexus" - default = "*pypi.org,files.pythonhosted.org,security.ubuntu.com,archive.ubuntu.com,keyserver.ubuntu.com,repo.anaconda.com,*.docker.com,*.docker.io,conda.anaconda.org,azure.archive.ubuntu.com, packages.microsoft.com" } From 0af829eaebf4939f52655fe48e8f348b6c9f0d7a Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 28 Apr 2022 14:59:20 +0000 Subject: [PATCH 085/142] Implemented app gateway start/stop --- .../shared_services/certs/scripts/letsencrypt.sh | 13 +++++++++++++ .../shared_services/certs/terraform/appgateway.tf | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/templates/shared_services/certs/scripts/letsencrypt.sh b/templates/shared_services/certs/scripts/letsencrypt.sh index 4339990f43..3ffd9db524 100755 --- a/templates/shared_services/certs/scripts/letsencrypt.sh +++ b/templates/shared_services/certs/scripts/letsencrypt.sh @@ -46,6 +46,15 @@ done # done with processing args and can set this set -o nounset +# Start the Application Gateway if stopped +echo "Checking app gateway status" +if [[ $(az network application-gateway list --output json --query "[?resourceGroup=='rg-${TRE_ID}'&&name=='agw-certs-${TRE_ID}'&&operationalState=='Stopped'] | length(@)") != 0 ]]; then + echo "App gateway stopped. Starting..." + az network application-gateway start -g "rg-$TRE_ID" -n "agw-certs-$TRE_ID" +else + echo "App gateway running" +fi + echo "Checking for index.html file in storage account" # Create the default index.html page @@ -133,3 +142,7 @@ az network application-gateway ssl-cert update \ --gateway-name "${application_gateway_name}" \ --name 'cert-primary' \ --key-vault-secret-id "${sid}" + +# Stop the app gateway once done to save cost +echo "Stopping app gateway" +az network application-gateway stop -g "rg-$TRE_ID" -n "agw-certs-$TRE_ID" diff --git a/templates/shared_services/certs/terraform/appgateway.tf b/templates/shared_services/certs/terraform/appgateway.tf index 4259d73c47..80afa5a7ec 100644 --- a/templates/shared_services/certs/terraform/appgateway.tf +++ b/templates/shared_services/certs/terraform/appgateway.tf @@ -160,4 +160,9 @@ resource "azurerm_application_gateway" "agw" { azurerm_key_vault_access_policy.app_gw_managed_identity ] + # Stop app gateway once provisioned to save cost until the generate custom action is invoked (which will start/stop as required) + provisioner "local-exec" { + command = "az network application-gateway stop -g ${data.azurerm_resource_group.rg.name} -n agw-certs-${var.tre_id}" + } + } From 5b61fc3b525893532e26593853ef2ba9fbb8ca71 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 28 Apr 2022 17:31:46 +0000 Subject: [PATCH 086/142] Separated cloudinit yaml into scripts --- ...gure_nexus.sh => configure_nexus_repos.sh} | 2 +- .../scripts/configure_nexus_ssl.sh | 69 +++++++++ .../apt-pypi_proxy_conf.json | 0 .../conda_forge_proxy_conf.json | 0 .../conda_proxy_conf.json | 0 .../docker_gpg_proxy_conf.json | 0 .../docker_hub_proxy_conf.json | 0 .../docker_proxy_conf.json | 0 .../pypi_proxy_conf.json | 0 .../ubuntu_proxy_conf.json | 0 .../ubuntu_security_proxy_conf.json | 0 .../scripts/reset_nexus_password.sh | 48 ++++++ .../terraform/cloud-config.yaml | 141 +----------------- .../sonatype-nexus/terraform/data.tf | 2 +- .../sonatype-nexus/terraform/vm.tf | 40 ++++- 15 files changed, 160 insertions(+), 142 deletions(-) rename templates/shared_services/sonatype-nexus/scripts/{configure_nexus.sh => configure_nexus_repos.sh} (96%) create mode 100644 templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/apt-pypi_proxy_conf.json (100%) rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/conda_forge_proxy_conf.json (100%) rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/conda_proxy_conf.json (100%) rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/docker_gpg_proxy_conf.json (100%) rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/docker_hub_proxy_conf.json (100%) rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/docker_proxy_conf.json (100%) rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/pypi_proxy_conf.json (100%) rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/ubuntu_proxy_conf.json (100%) rename templates/shared_services/sonatype-nexus/scripts/{nexus_config => nexus_repos_config}/ubuntu_security_proxy_conf.json (100%) create mode 100644 templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh similarity index 96% rename from templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh rename to templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh index 433af77c21..d685dcbfa6 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh @@ -49,7 +49,7 @@ if [ -z "$NEXUS_PASS" ]; then fi # Create proxy for each .json file -for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_config/*.json; do +for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_repos_config/*.json; do echo "Found config file: $filename. Sending to Nexus..." # Check if apt proxy base_type=$( jq .baseType "$filename" | sed 's/"//g') diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh new file mode 100644 index 0000000000..30f55bbfb7 --- /dev/null +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Configure Nexus to use certificate to serve proxies over https + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace + +# Prepare ssl certificate +az login --identity -u "${msi_id}" --allow-no-subscriptions +# -- get cert from kv as secret so it contains private key +echo 'Getting cert and cert password from Keyvault...' +az keyvault secret download --vault-name "${vault_name}" --name "${ssl_cert_name}" --file temp.pfx --encoding base64 +CERT_PASSWORD=$(az keyvault secret show --vault-name "${vault_name}" \ + --name "${ssl_cert_password_name}" -o tsv --query value) +# -- az cli strips out password from cert so we re-add by converting to PEM then PFX with pwd +openssl pkcs12 -in temp.pfx -out temp.pem -nodes -password pass: +openssl pkcs12 -export -out nexus-ssl.pfx -in temp.pem -password "pass:$CERT_PASSWORD" + +# Import ssl cert to keystore within Nexus volume +keystore_timeout=300 +echo 'Checking for nexus-data/keystores directory...' +while [ ! -d /etc/nexus-data/keystores ]; do + # Wait for /keystore dir to be created by container first + if [ $keystore_timeout == 0 ]; then + echo 'ERROR - Timeout while waiting for Nexus to create nexus-data/keystores' + exit 1 + fi + sleep 1 + ((keystore_timeout--)) +done +echo 'Directory found. Importing ssl cert into nexus-data/keystores/keystore.jks...' +keytool -v -importkeystore -noprompt -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 \ + -destkeystore /etc/nexus-data/keystores/keystore.jks \ + -deststoretype JKS -srcstorepass "$CERT_PASSWORD" -deststorepass "$CERT_PASSWORD" + +# Configure Jetty instance within Nexus to consume ssl cert +echo 'Modifying Nexus Jetty configuration to enable ssl...' +mkdir -p /etc/nexus-data/etc/jetty +# -- first need to copy default Jetty config to persistent volume so isn't overwritten on restart +docker exec -u root nexus cp /opt/sonatype/nexus/etc/jetty/jetty-https.xml /nexus-data/etc/jetty/ +# -- then we replace password values with the ssl cert keystore password +xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePassword']" \ + -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml +xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyManagerPassword']" \ + -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml +xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePassword']" \ + -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml +# -- then update the location of our keystore +xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePath']" \ + -v /nexus-data/keystores/keystore.jks /etc/nexus-data/etc/jetty/jetty-https.xml +xmlstarlet ed -P --inplace \ + -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePath']" \ + -v /nexus-data/keystores/keystore.jks /etc/nexus-data/etc/jetty/jetty-https.xml + +# Add jetty configuration and ssl port to Nexus properties +cat >> /etc/nexus-data/etc/nexus.properties <<'EOF' +application-port-ssl=8443 +nexus-args=$${jetty.etc}/jetty.xml,$${jetty.etc}/jetty-http.xml,$${jetty.etc}/jetty-requestlog.xml,/nexus-data/etc/jetty/jetty-https.xml +EOF + +# Restart the container for changes to take effect +docker restart nexus +echo 'Nexus ssl configuration completed.' diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/apt-pypi_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/apt-pypi_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/apt-pypi_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/apt-pypi_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/conda_forge_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/conda_forge_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/conda_forge_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/conda_forge_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/conda_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/conda_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/conda_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/conda_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_gpg_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_gpg_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_gpg_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_gpg_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_hub_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_hub_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_hub_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_hub_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/pypi_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/pypi_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/pypi_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/pypi_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/ubuntu_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/ubuntu_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/ubuntu_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/ubuntu_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_config/ubuntu_security_proxy_conf.json b/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_config/ubuntu_security_proxy_conf.json rename to templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh b/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh new file mode 100644 index 0000000000..6932478ba8 --- /dev/null +++ b/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace + +if [ -z "$1" ] + then + echo "New password to set needs to be passed as argument" +fi + +# Get the current password so we can post to the API +# (this is created in /nexus-data mounted volume as part of Nexus container start-up) +password_timeout=300 +echo 'Checking for Nexus admin password file...' +while [ ! -f /etc/nexus-data/admin.password ]; do + # We must first wait for the file to be created + if [ $password_timeout == 0 ]; then + echo 'ERROR - Timeout while waiting for nexus-data/admin.password to be created' + exit 1 + fi + sleep 1 + ((password_timeout--)) +done +CURRENT_PASSWORD=$(cat /etc/nexus-data/admin.password) + +# Set own admin password so we can connect to repository manager later on using TF KV secret +reset_timeout=300 +echo "Nexus default admin password found ($CURRENT_PASSWORD). Resetting..." +# While the container is starting up it may return a number of transient errors which we need to retry +# NOTE: we can't use curl's built-in retry flags as it doesn't catch for the connection reset response +res=1 +while test "$res" != "0"; do + curl -ifu admin:"$CURRENT_PASSWORD" -XPUT -H 'Content-Type:text/plain' --data "$1" \ + http://localhost/service/rest/v1/security/users/admin/change-password + res=$? + echo "Attempt to reset password finished with code $res" + if test "$res" == "0"; then + echo 'Password reset successfully. Admin can now log in with secret stored in KeyVault.' + else + if [ $reset_timeout == 0 ]; then + echo 'ERROR - Timeout while trying to reset Nexus admin password' + exit 1 + fi + sleep 5 + ((reset_timeout+=5)) + fi +done diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 3d31e81c37..43f9f6c48d 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -34,141 +34,6 @@ system_info: default_user: groups: [docker] -write_files: - - path: /etc/cron.daily/renew-nexus-cert - # Daily cron job to renew nexus cert based on certificate present in keyvault - content: | - #!/bin/bash - set -o errexit - echo "Calling configure-nexus-ssl script to renew ssl certificate" - bash /home/adminuser/configure-nexus-ssl.sh - permissions: '0755' - - # Configure Nexus to disable default repos - - path: /etc/nexus-data/etc/nexus.properties - content: | - nexus.skipDefaultRepositories=true - permissions: '0755' - - # Set up Nexus to serve https using SSL cert - - path: /home/adminuser/configure-nexus-ssl.sh - content: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o nounset - # set -o xtrace - - # Prepare ssl certificate - az login --identity -u ${msi_id} --allow-no-subscriptions - # -- get cert from kv as secret so it contains private key - echo 'Getting cert and cert password from Keyvault...' - az keyvault secret download --vault-name ${vault_name} --name ${ssl_cert_name} --file temp.pfx --encoding base64 - CERT_PASSWORD=$(az keyvault secret show --vault-name ${vault_name} \ - --name ${ssl_cert_password_name} -o tsv --query value) - # -- az cli strips out password from cert so we re-add by converting to PEM then PFX with pwd - openssl pkcs12 -in temp.pfx -out temp.pem -nodes -password pass: - openssl pkcs12 -export -out nexus-ssl.pfx -in temp.pem -password "pass:$CERT_PASSWORD" - - # Import ssl cert to keystore within Nexus volume - keystore_timeout=300 - echo 'Checking for nexus-data/keystores directory...' - while [ ! -d /etc/nexus-data/keystores ]; do - # Wait for /keystore dir to be created by container first - if [ $keystore_timeout == 0 ]; then - echo 'ERROR - Timeout while waiting for Nexus to create nexus-data/keystores' - exit 1 - fi - sleep 1 - ((keystore_timeout--)) - done - echo 'Directory found. Importing ssl cert into nexus-data/keystores/keystore.jks...' - keytool -v -importkeystore -noprompt -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 \ - -destkeystore /etc/nexus-data/keystores/keystore.jks \ - -deststoretype JKS -srcstorepass "$CERT_PASSWORD" -deststorepass "$CERT_PASSWORD" - - # Configure Jetty instance within Nexus to consume ssl cert - echo 'Modifying Nexus Jetty configuration to enable ssl...' - mkdir -p /etc/nexus-data/etc/jetty - # -- first need to copy default Jetty config to persistent volume so isn't overwritten on restart - docker exec -u root nexus cp /opt/sonatype/nexus/etc/jetty/jetty-https.xml /nexus-data/etc/jetty/ - # -- then we replace password values with the ssl cert keystore password - xmlstarlet ed -P --inplace \ - -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePassword']" \ - -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml - xmlstarlet ed -P --inplace \ - -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyManagerPassword']" \ - -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml - xmlstarlet ed -P --inplace \ - -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePassword']" \ - -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml - # -- then update the location of our keystore - xmlstarlet ed -P --inplace \ - -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePath']" \ - -v /nexus-data/keystores/keystore.jks /etc/nexus-data/etc/jetty/jetty-https.xml - xmlstarlet ed -P --inplace \ - -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePath']" \ - -v /nexus-data/keystores/keystore.jks /etc/nexus-data/etc/jetty/jetty-https.xml - - # Add jetty configuration and ssl port to Nexus properties - cat >> /etc/nexus-data/etc/nexus.properties <<'EOF' - application-port-ssl=8443 - nexus-args=$${jetty.etc}/jetty.xml,$${jetty.etc}/jetty-http.xml,$${jetty.etc}/jetty-requestlog.xml,/nexus-data/etc/jetty/jetty-https.xml - EOF - - # Restart the container for changes to take effect - docker restart nexus - echo 'Nexus ssl configuration completed.' - permissions: '0744' - - # Write a script that will reset the admin password for Nexus to the one TF generated - - path: /home/adminuser/reset-nexus-password.sh - content: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o nounset - # set -o xtrace - - # Get the current password so we can post to the API - # (this is created in /nexus-data mounted volume as part of Nexus container start-up) - password_timeout=300 - echo 'Checking for Nexus admin password file...' - while [ ! -f /etc/nexus-data/admin.password ]; do - # We must first wait for the file to be created - if [ $password_timeout == 0 ]; then - echo 'ERROR - Timeout while waiting for nexus-data/admin.password to be created' - exit 1 - fi - sleep 1 - ((password_timeout--)) - done - CURRENT_PASSWORD=$(cat /etc/nexus-data/admin.password) - - # Set own admin password so we can connect to repository manager later on using TF KV secret - reset_timeout=300 - echo "Nexus default admin password found ($CURRENT_PASSWORD). Resetting..." - # While the container is starting up it may return a number of transient errors which we need to retry - # NOTE: we can't use curl's built-in retry flags as it doesn't catch for the connection reset response - res=1 - while test "$res" != "0"; do - curl -ifu admin:$CURRENT_PASSWORD -XPUT -H 'Content-Type:text/plain' --data '${nexus_admin_password}' \ - http://localhost/service/rest/v1/security/users/admin/change-password - res=$? - echo "Attempt to reset password finished with code $res" - if test "$res" == "0"; then - echo 'Password reset successfully. Admin can now log in with secret stored in KeyVault.' - else - if [ $reset_timeout == 0 ]; then - echo 'ERROR - Timeout while trying to reset Nexus admin password' - exit 1 - fi - sleep 5 - ((reset_timeout+=5)) - fi - done - permissions: '0744' - runcmd: - export DEBIAN_FRONTEND=noninteractive # Give the Nexus process write permissions on the folder mounted as persistent volume @@ -179,5 +44,7 @@ runcmd: --name nexus --log-driver local sonatype/nexus3 - - bash /home/adminuser/reset-nexus-password.sh - - bash /home/adminuser/configure-nexus-ssl.sh + # Reset the admin password of Nexus to the one created by TF and stored in KeyVault + - bash /home/adminuser/reset_nexus_password.sh "${nexus_admin_password}" + # Invoke Nexus SSL configuration (which will also be ran as CRON daily to renew cert) + - bash /etc/cron.daily/configure_nexus_ssl.sh diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus/terraform/data.tf index a806a131ec..fc5031c4ce 100644 --- a/templates/shared_services/sonatype-nexus/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus/terraform/data.tf @@ -30,7 +30,7 @@ data "azurerm_key_vault_certificate" "nexus_cert" { } data "azurerm_key_vault_secret" "nexus_cert_password" { - name = "nexus-letsencrypt-cert-password" + name = "${data.azurerm_key_vault_certificate.nexus_cert.name}-password" key_vault_id = data.azurerm_key_vault.kv.id } diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 61b3e80c6b..06ac0fb61c 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -133,14 +133,48 @@ data "template_cloudinit_config" "nexus_config" { part { content_type = "text/cloud-config" - content = data.template_file.nexus_config.rendered + content = data.template_file.nexus_bootstrapping.rendered + } + + part { + content_type = "text/cloud-config" + content = jsonencode({ + write_files = [ + { + content = file("${path.module}/../scripts/configure_nexus_repos.sh") + path = "/home/adminuser/configure_nexus_repos.sh" + permissions = "0744" + }, + { + content = data.template_file.configure_nexus_ssl.rendered + path = "/etc/cron.daily/configure_nexus_ssl.sh" + permissions = "0755" + }, + { + content = "nexus.skipDefaultRepositories=true" + path = "/etc/nexus-data/etc/nexus.properties" + permissions = "0755" + }, + { + content = file("${path.module}/../scripts/reset_nexus_password.sh") + path = "/home/adminuser/reset_nexus_password.sh" + permissions = "0744" + } + ] + }) } } -data "template_file" "nexus_config" { +data "template_file" "nexus_bootstrapping" { template = file("${path.module}/cloud-config.yaml") vars = { - nexus_admin_password = random_password.nexus_admin_password.result + nexus_admin_password = random_password.nexus_admin_password.result + } +} + +data "template_file" "configure_nexus_ssl" { + template = file("${path.module}/../scripts/configure_nexus_ssl.sh") + vars = { msi_id = azurerm_user_assigned_identity.nexus_msi.id vault_name = data.azurerm_key_vault.kv.name ssl_cert_name = data.azurerm_key_vault_certificate.nexus_cert.name From 5dbbb1588431e4f455d956330d099116cc9d6e1b Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 28 Apr 2022 19:13:12 +0000 Subject: [PATCH 087/142] Fixed new line issue --- templates/shared_services/sonatype-nexus/terraform/vm.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 06ac0fb61c..f5b7c8ac51 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -151,7 +151,7 @@ data "template_cloudinit_config" "nexus_config" { permissions = "0755" }, { - content = "nexus.skipDefaultRepositories=true" + content = "nexus.skipDefaultRepositories=true\n" path = "/etc/nexus-data/etc/nexus.properties" permissions = "0755" }, From 6d837b7ae5b91056c7e8f1c0b1e1acdb1db70229 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 28 Apr 2022 19:27:04 +0000 Subject: [PATCH 088/142] Fixed bash casing --- .../scripts/configure_nexus_repos.sh | 20 +++++++++---------- .../scripts/configure_nexus_ssl.sh | 18 ++++++++--------- .../scripts/reset_nexus_password.sh | 6 +++--- .../terraform/cloud-config.yaml | 4 ++-- .../sonatype-nexus/terraform/vm.tf | 10 +++++----- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh index d685dcbfa6..4277f26ffd 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh @@ -38,12 +38,12 @@ while [ "$1" != "" ]; do shift # remove the current value for `$1` and use the next done -NEXUS_URL="https://nexus-${tre_id}.${location}.cloudapp.azure.com" -NEXUS_ADMIN_PASSWORD_NAME="nexus-admin-password" -KEYVAULT_NAME="kv-${tre_id}" -NEXUS_PASS=$(az keyvault secret show --name "${NEXUS_ADMIN_PASSWORD_NAME}" --vault-name "${KEYVAULT_NAME}" -o json | jq -r '.value') +nexus_url="https://nexus-${tre_id}.${location}.cloudapp.azure.com" +nexus_admin_password_name="nexus-admin-password" +keyvault_name="kv-${tre_id}" +nexus_pass=$(az keyvault secret show --name "${nexus_admin_password_name}" --vault-name "${keyvault_name}" -o json | jq -r '.value') -if [ -z "$NEXUS_PASS" ]; then +if [ -z "$nexus_pass" ]; then echo "Unable to get the Nexus admin password from Keyvault. You may need to manually reset it in the Nexus host. Refer to the public Nexus documentation for more information." exit 1 fi @@ -56,15 +56,15 @@ for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_repos_config/*.json; do repo_type=$( jq .repoType "$filename" | sed 's/"//g') repo_name=$(jq .name "$filename" | sed 's/"//g') - base_url=$NEXUS_URL/service/rest/v1/repositories/$base_type/$repo_type + base_url=$nexus_url/service/rest/v1/repositories/$base_type/$repo_type full_url=$base_url/$repo_name - STATUS_CODE=$(curl -iu admin:"$NEXUS_PASS" -X "GET" "$full_url" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) - echo "Response received from Nexus: $STATUS_CODE" + status_code=$(curl -iu admin:"$nexus_pass" -X "GET" "$full_url" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) + echo "Response received from Nexus: $status_code" - if [[ ${STATUS_CODE} == 404 ]] + if [[ ${status_code} == 404 ]] then - curl -iu admin:"$NEXUS_PASS" -XPOST \ + curl -iu admin:"$nexus_pass" -XPOST \ "$base_url" \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh index 30f55bbfb7..9c57e86323 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh @@ -8,15 +8,15 @@ set -o nounset # set -o xtrace # Prepare ssl certificate -az login --identity -u "${msi_id}" --allow-no-subscriptions +az login --identity -u "${MSI_ID}" --allow-no-subscriptions # -- get cert from kv as secret so it contains private key echo 'Getting cert and cert password from Keyvault...' -az keyvault secret download --vault-name "${vault_name}" --name "${ssl_cert_name}" --file temp.pfx --encoding base64 -CERT_PASSWORD=$(az keyvault secret show --vault-name "${vault_name}" \ - --name "${ssl_cert_password_name}" -o tsv --query value) +az keyvault secret download --vault-name "${VAULT_NAME}" --name "${SSL_CERT_NAME}" --file temp.pfx --encoding base64 +cert_password=$(az keyvault secret show --vault-name "${VAULT_NAME}" \ + --name "${SSL_CERT_PASSWORD_NAME}" -o tsv --query value) # -- az cli strips out password from cert so we re-add by converting to PEM then PFX with pwd openssl pkcs12 -in temp.pfx -out temp.pem -nodes -password pass: -openssl pkcs12 -export -out nexus-ssl.pfx -in temp.pem -password "pass:$CERT_PASSWORD" +openssl pkcs12 -export -out nexus-ssl.pfx -in temp.pem -password "pass:$cert_password" # Import ssl cert to keystore within Nexus volume keystore_timeout=300 @@ -33,7 +33,7 @@ done echo 'Directory found. Importing ssl cert into nexus-data/keystores/keystore.jks...' keytool -v -importkeystore -noprompt -srckeystore nexus-ssl.pfx -srcstoretype PKCS12 \ -destkeystore /etc/nexus-data/keystores/keystore.jks \ - -deststoretype JKS -srcstorepass "$CERT_PASSWORD" -deststorepass "$CERT_PASSWORD" + -deststoretype JKS -srcstorepass "$cert_password" -deststorepass "$cert_password" # Configure Jetty instance within Nexus to consume ssl cert echo 'Modifying Nexus Jetty configuration to enable ssl...' @@ -43,13 +43,13 @@ docker exec -u root nexus cp /opt/sonatype/nexus/etc/jetty/jetty-https.xml /nexu # -- then we replace password values with the ssl cert keystore password xmlstarlet ed -P --inplace \ -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePassword']" \ - -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml + -v "$cert_password" /etc/nexus-data/etc/jetty/jetty-https.xml xmlstarlet ed -P --inplace \ -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyManagerPassword']" \ - -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml + -v "$cert_password" /etc/nexus-data/etc/jetty/jetty-https.xml xmlstarlet ed -P --inplace \ -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='TrustStorePassword']" \ - -v "$CERT_PASSWORD" /etc/nexus-data/etc/jetty/jetty-https.xml + -v "$cert_password" /etc/nexus-data/etc/jetty/jetty-https.xml # -- then update the location of our keystore xmlstarlet ed -P --inplace \ -u "/Configure[@id='Server']/New[@id='sslContextFactory']/Set[@name='KeyStorePath']" \ diff --git a/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh b/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh index 6932478ba8..ec44e8e8f4 100644 --- a/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh +++ b/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh @@ -22,16 +22,16 @@ while [ ! -f /etc/nexus-data/admin.password ]; do sleep 1 ((password_timeout--)) done -CURRENT_PASSWORD=$(cat /etc/nexus-data/admin.password) +current_password=$(cat /etc/nexus-data/admin.password) # Set own admin password so we can connect to repository manager later on using TF KV secret reset_timeout=300 -echo "Nexus default admin password found ($CURRENT_PASSWORD). Resetting..." +echo "Nexus default admin password found ($current_password). Resetting..." # While the container is starting up it may return a number of transient errors which we need to retry # NOTE: we can't use curl's built-in retry flags as it doesn't catch for the connection reset response res=1 while test "$res" != "0"; do - curl -ifu admin:"$CURRENT_PASSWORD" -XPUT -H 'Content-Type:text/plain' --data "$1" \ + curl -ifu admin:"$current_password" -XPUT -H 'Content-Type:text/plain' --data "$1" \ http://localhost/service/rest/v1/security/users/admin/change-password res=$? echo "Attempt to reset password finished with code $res" diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index 43f9f6c48d..5c5b07b425 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -44,7 +44,7 @@ runcmd: --name nexus --log-driver local sonatype/nexus3 - # Reset the admin password of Nexus to the one created by TF and stored in KeyVault - - bash /home/adminuser/reset_nexus_password.sh "${nexus_admin_password}" # Invoke Nexus SSL configuration (which will also be ran as CRON daily to renew cert) - bash /etc/cron.daily/configure_nexus_ssl.sh + # Reset the admin password of Nexus to the one created by TF and stored in KeyVault + - bash /home/adminuser/reset_nexus_password.sh "${NEXUS_ADMIN_PASSWORD}" diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index f5b7c8ac51..1a2c4cb29c 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -168,16 +168,16 @@ data "template_cloudinit_config" "nexus_config" { data "template_file" "nexus_bootstrapping" { template = file("${path.module}/cloud-config.yaml") vars = { - nexus_admin_password = random_password.nexus_admin_password.result + NEXUS_ADMIN_PASSWORD = random_password.nexus_admin_password.result } } data "template_file" "configure_nexus_ssl" { template = file("${path.module}/../scripts/configure_nexus_ssl.sh") vars = { - msi_id = azurerm_user_assigned_identity.nexus_msi.id - vault_name = data.azurerm_key_vault.kv.name - ssl_cert_name = data.azurerm_key_vault_certificate.nexus_cert.name - ssl_cert_password_name = data.azurerm_key_vault_secret.nexus_cert_password.name + MSI_ID = azurerm_user_assigned_identity.nexus_msi.id + VAULT_NAME = data.azurerm_key_vault.kv.name + SSL_CERT_NAME = data.azurerm_key_vault_certificate.nexus_cert.name + SSL_CERT_PASSWORD_NAME = data.azurerm_key_vault_secret.nexus_cert_password.name } } From 587c1cd40909d5c7e66843ec959a54da3937ea05 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 28 Apr 2022 21:37:53 +0000 Subject: [PATCH 089/142] Added local nexus repo config --- .../scripts/configure_nexus_repos.sh | 54 +++---------------- .../scripts/reset_nexus_password.sh | 29 +++++----- .../terraform/cloud-config.yaml | 2 + .../sonatype-nexus/terraform/vm.tf | 6 +++ 4 files changed, 27 insertions(+), 64 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh index 4277f26ffd..bc2e839374 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh @@ -1,51 +1,9 @@ #!/bin/bash set -e -function usage() { - cat < Date: Fri, 29 Apr 2022 14:41:22 +0000 Subject: [PATCH 090/142] Added retry logic to config repos --- .../scripts/configure_nexus_repos.sh | 44 +++++++++++++------ .../scripts/reset_nexus_password.sh | 30 ++++++------- .../terraform/cloud-config.yaml | 7 +-- .../sonatype-nexus/terraform/vm.tf | 24 ++++++---- 4 files changed, 66 insertions(+), 39 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh index bc2e839374..09df0ba706 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh @@ -1,11 +1,25 @@ #!/bin/bash -set -e +set -o pipefail +set -o nounset +# set -o xtrace if [ -z "$1" ] then echo 'Nexus password needs to be passed as argument' fi +timeout=300 +echo 'Checking for ./nexus_repos_config directory...' +while [ ! -d "$(dirname "${BASH_SOURCE[0]}")"/nexus_repos_config ]; do + # Wait for /nexus_repos_config with json config files to be copied into vm + if [ $timeout == 0 ]; then + echo 'ERROR - Timeout while waiting for nexus_repos_config directory' + exit 1 + fi + sleep 1 + ((timeout--)) +done + # Create proxy for each .json file for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_repos_config/*.json; do echo "Found config file: $filename. Sending to Nexus..." @@ -13,21 +27,25 @@ for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_repos_config/*.json; do base_type=$( jq .baseType "$filename" | sed 's/"//g') repo_type=$( jq .repoType "$filename" | sed 's/"//g') repo_name=$(jq .name "$filename" | sed 's/"//g') - base_url=http://localhost/service/rest/v1/repositories/$base_type/$repo_type - full_url=$base_url/$repo_name - status_code=$(curl -iu admin:"$1" -X "GET" "$full_url" -H "accept: application/json" -k -s -w "%{http_code}" -o /dev/null) - echo "Response received from Nexus: $status_code" - - if [[ ${status_code} == 404 ]] - then - curl -iu admin:"$1" -XPOST \ + config_timeout=300 + status_code=1 + while [ $status_code != 200 ]; do + status_code=$(curl -iu admin:"$1" -XPOST \ "$base_url" \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ - -d @"$filename" - else - echo "$repo_type proxy for $repo_name already exists." - fi + -d @"$filename" \ + -k -s -w "%{http_code}" -o /dev/null) + echo "Response received from Nexus: $status_code" + + if [ $config_timeout == 0 ]; then + echo "ERROR - Timeout while trying to configure $repo_name" + exit 1 + elif [ "$status_code" != 200 ]; then + sleep 1 + ((config_timeout--)) + fi + done done diff --git a/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh b/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh index 635767b247..8383d6d6dc 100644 --- a/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh +++ b/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -o errexit set -o pipefail set -o nounset # set -o xtrace @@ -27,19 +26,20 @@ current_password=$(cat /etc/nexus-data/admin.password) # Set own admin password so we can connect to repository manager later on using TF KV secret reset_timeout=300 echo "Nexus default admin password found ($current_password). Resetting..." -# While the container is starting up it may return a number of transient errors which we need to retry -# NOTE: we can't use curl's built-in retry flags as it doesn't catch for the connection reset response -status=1 -while [ ${status} != "204" ]; do - status=$(curl -ifu admin:"$current_password" -XPUT -H 'Content-Type:text/plain' --data "$1" \ - -s -o /dev/null -L -w "%{http_code}" http://localhost/service/rest/v1/security/users/admin/change-password) - echo "Attempt to reset password finished with code $status" - if [ $reset_timeout == 0 ]; then - echo 'ERROR - Timeout while trying to reset Nexus admin password' - exit 1 +res=1 +while test "$res" != "0"; do + curl -ifu admin:"$current_password" -XPUT -H 'Content-Type:text/plain' --data "$1" \ + http://localhost/service/rest/v1/security/users/admin/change-password + res=$? + echo "Attempt to reset password finished with code $res" + if test "$res" == "0"; then + echo 'Password reset successfully.' + else + if [ $reset_timeout == 0 ]; then + echo 'ERROR - Timeout while trying to reset Nexus admin password' + exit 1 + fi + sleep 5 + ((reset_timeout+=5)) fi - sleep 5 - ((reset_timeout+=5)) done - -echo 'Password reset successfully.' diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml index f31e2bc7a1..336cc95ae4 100644 --- a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml +++ b/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml @@ -24,6 +24,7 @@ packages: - azure-cli - default-jre - xmlstarlet + - jq # create the docker group groups: @@ -44,9 +45,9 @@ runcmd: --name nexus --log-driver local sonatype/nexus3 + # Reset the admin password of Nexus to the one created by TF and stored in KeyVault + - bash /tmp/reset_nexus_password.sh "${NEXUS_ADMIN_PASSWORD}" # Invoke Nexus SSL configuration (which will also be ran as CRON daily to renew cert) - bash /etc/cron.daily/configure_nexus_ssl.sh - # Reset the admin password of Nexus to the one created by TF and stored in KeyVault - - bash /home/adminuser/reset_nexus_password.sh "${NEXUS_ADMIN_PASSWORD}" # Configure Nexus repositories - - bash /home/adminuser/configure_nexus_repos.sh "${NEXUS_ADMIN_PASSWORD}" + - bash /tmp/configure_nexus_repos.sh "${NEXUS_ADMIN_PASSWORD}" diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index d59d5e923f..6c86d97aca 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -125,6 +125,20 @@ resource "azurerm_linux_virtual_machine" "nexus" { azurerm_key_vault_access_policy.nexus_msi, azurerm_firewall_application_rule_collection.shared_subnet_nexus ] + + connection { + type = "ssh" + host = "${azurerm_network_interface.nexus.private_ip_address}" + user = "adminuser" + password = random_password.nexus_vm_password.result + agent = false + timeout = "10m" + } + + provisioner "file" { + source = "${path.module}/../scripts/nexus_repos_config" + destination = "/tmp/nexus_repos_config" + } } data "template_cloudinit_config" "nexus_config" { @@ -142,7 +156,7 @@ data "template_cloudinit_config" "nexus_config" { write_files = [ { content = file("${path.module}/../scripts/configure_nexus_repos.sh") - path = "/home/adminuser/configure_nexus_repos.sh" + path = "/tmp/configure_nexus_repos.sh" permissions = "0744" }, { @@ -157,13 +171,7 @@ data "template_cloudinit_config" "nexus_config" { }, { content = file("${path.module}/../scripts/reset_nexus_password.sh") - path = "/home/adminuser/reset_nexus_password.sh" - permissions = "0744" - }, - { - for_each = fileset("${path.module}/../scripts/nexus_repos_config", "*.json") - content = "${each.value}" - path = "/home/adminuser/nexus_repos_config/${each.key}" + path = "/tmp/reset_nexus_password.sh" permissions = "0744" } ] From 76f4d876ce2e891d6c547e2dee945fe5bdff32ee Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 29 Apr 2022 14:46:15 +0000 Subject: [PATCH 091/142] gitea bump --- templates/shared_services/gitea/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/gitea/version.txt b/templates/shared_services/gitea/version.txt index f1380eede2..9cb17e7976 100644 --- a/templates/shared_services/gitea/version.txt +++ b/templates/shared_services/gitea/version.txt @@ -1 +1 @@ -__version__ = "0.1.7" +__version__ = "0.1.8" From 41d6ac037a46b4f85263cfc9eb9abf6c128527a8 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 29 Apr 2022 16:33:05 +0000 Subject: [PATCH 092/142] Fixed status code --- .../sonatype-nexus/scripts/configure_nexus_repos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh index 09df0ba706..2a148bb2d3 100644 --- a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh @@ -31,7 +31,7 @@ for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_repos_config/*.json; do config_timeout=300 status_code=1 - while [ $status_code != 200 ]; do + while [ $status_code != 201 ]; do status_code=$(curl -iu admin:"$1" -XPOST \ "$base_url" \ -H 'accept: application/json' \ @@ -43,7 +43,7 @@ for filename in "$(dirname "${BASH_SOURCE[0]}")"/nexus_repos_config/*.json; do if [ $config_timeout == 0 ]; then echo "ERROR - Timeout while trying to configure $repo_name" exit 1 - elif [ "$status_code" != 200 ]; then + elif [ "$status_code" != 201 ]; then sleep 1 ((config_timeout--)) fi From 3574043cb03e5eb0c170506da4d3a2ca5312d9d6 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 29 Apr 2022 16:37:13 +0000 Subject: [PATCH 093/142] terraform linting --- .../shared_services/sonatype-nexus/terraform/variables.tf | 4 ++-- templates/shared_services/sonatype-nexus/terraform/vm.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus/terraform/variables.tf index d63fda333f..604381e20d 100644 --- a/templates/shared_services/sonatype-nexus/terraform/variables.tf +++ b/templates/shared_services/sonatype-nexus/terraform/variables.tf @@ -1,7 +1,7 @@ variable "tre_id" { - type = string + type = string } variable "ssl_cert_name" { - type = string + type = string } diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus/terraform/vm.tf index 6c86d97aca..fb88d44817 100644 --- a/templates/shared_services/sonatype-nexus/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus/terraform/vm.tf @@ -128,7 +128,7 @@ resource "azurerm_linux_virtual_machine" "nexus" { connection { type = "ssh" - host = "${azurerm_network_interface.nexus.private_ip_address}" + host = azurerm_network_interface.nexus.private_ip_address user = "adminuser" password = random_password.nexus_vm_password.result agent = false From f31f5fbafaee9ba06abf999b497b2930002729c8 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 29 Apr 2022 17:55:06 +0000 Subject: [PATCH 094/142] Added docs --- .../configuring-shared-services.md | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 55bb1563b0..58e3374079 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -2,14 +2,71 @@ Complete the configuration of the shared services (Nexus and Gitea) from inside of the TRE environment. -Make sure you run the following command using git bash and set your current directory as C:/AzureTRE +## Configure Nexus -## Configure Nexus repository proxies +Before deploying the Nexus shared service, you need to make sure that it will have access to a certificate to configure serving proxies over https with. By default, the Nexus service will serve proxies from `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`, and thus it requires a certificate that validates ownership of this domain to use for SSL. -1. Run the Nexus configuration script to reset the password and set up several common repository proxies on Nexus. Substitute `` with the TRE_ID you chose for the core deployment and `` with the Azure region you deployed to: -```./templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh -t -l ``` +You can use the Certs Shared Service to set one up by following these steps: -You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/` in the jumpbox and signing in with the username `admin` and the password secret located in your core keyvault, with the key `nexus-admin-password`. +1. Run the below commands in your terminal to build, publish and register the certs bundle: + + ```cmd + make bundle-build DIR=./templates/shared_services/certs + make bundle-publish DIR=./templates/shared_services/certs + make bundle-register DIR=./templates/shared_services/certs BUNDLE_TYPE=shared_service + ``` + +1. Navigate to the Swagger UI for your TRE API at `https:///api/docs`, and authenticate if you haven't already by clicking `Authorize`. + +1. Click `Try it out` on the `POST` `/api/shared-services` operation, and paste the following to deploy the certs service: + + ```json + { + "templateName": "tre-shared-service-certs", + "properties": { + "display_name": "Nexus cert", + "description": "Generate/renew ssl cert for Nexus shared service", + "domain_prefix": "nexus", + "cert_name": "nexus-ssl" + } + } + ``` + +1. Once the shared service has been deployed (which you can check by querying the `/api/shared-services/operations` method), copy its `resource_id`, then find the `POST` operation for `/api/shared-services/{shared_service_id}/invoke_action`, click `Try it out` and paste in the resource id into the `shared_service_id` field, and enter `generate` into the `action` field, then click `Execute`. + +This will invoke the certs service to use Letsencrypt to generate a certificate for the specified domain prefix followed by `-{TRE_ID}.{LOCATION}.cloudapp.azure.com`, so in our case, having entered `nexus`, this will be `nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`, which will be the public domain for our Nexus service. + +Once this has completed, you can verify its success either from the operation output, or by navigating to your core keyvault (`kv-{TRE_ID}`) and looking for a certificate called `nexus-ssl`. + +After verifying the certificate has been generated, you can deploy Nexus: + +1. Run the below commands in your terminal to build, publish and register the Nexus shared service bundle: + + ```cmd + make bundle-build DIR=./templates/shared_services/certs + make bundle-publish DIR=./templates/shared_services/certs + make bundle-register DIR=./templates/shared_services/certs BUNDLE_TYPE=shared_service + ``` + +1. Navigate to the Swagger UI for your TRE API at `https:///api/docs`, and authenticate if you haven't already by clicking `Authorize`. + +1. Click `Try it out` on the `POST` `/api/shared-services` operation, and paste the following to deploy the Nexus shared service: + + ```json + { + "templateName": "tre-shared-service-nexus", + "properties": { + "display_name": "Nexus", + "description": "Proxy public repositories with Nexus" + } + } + ``` + +This will deploy the infrastructure required for Nexus, then start the service and configure it with the repository configurations located in the `./templates/shared_services/sonatype-nexus/scripts/nexus_repos_config` folder. It will also set up HTTPS using the certificate you generated in the previous section, so proxies can be served at `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`. + +You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/` in the jumpbox and signing in with the username `admin` and the password secret located in your core keyvault, with the key `nexus-admin-password`. Here you should be able to see all of the configured repositories and you can use the UI to manage settings etc. + +Just bear in mind that if this service is redeployed any changes in the UI won't be persisted. If you wish to add new repositories or alter existing ones, use the JSON files within the `./nexus_repos_config` directory. ## Configure Gitea repositories @@ -19,6 +76,7 @@ By default, this Gitea instance does not have any repositories configured. You c ### Command Line +Make sure you run the following commands using git bash and set your current directory as C:/AzureTRE. 1. On the jumbox, run: ```./scripts/gitea_migrate_repo.sh -t -g ``` From 52d0df70c1d272a3080d72a6b59efd67565ccdd2 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 29 Apr 2022 18:05:50 +0000 Subject: [PATCH 095/142] Lint fix --- .../setup-instructions/configuring-shared-services.md | 2 +- templates/shared_services/certs/porter.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 58e3374079..97a64d6741 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -64,7 +64,7 @@ After verifying the certificate has been generated, you can deploy Nexus: This will deploy the infrastructure required for Nexus, then start the service and configure it with the repository configurations located in the `./templates/shared_services/sonatype-nexus/scripts/nexus_repos_config` folder. It will also set up HTTPS using the certificate you generated in the previous section, so proxies can be served at `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`. -You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/` in the jumpbox and signing in with the username `admin` and the password secret located in your core keyvault, with the key `nexus-admin-password`. Here you should be able to see all of the configured repositories and you can use the UI to manage settings etc. +You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/` in the jumpbox and signing in with the username `admin` and the password secret located in your core keyvault, with the key `nexus-admin-password`. Here you should be able to see all of the configured repositories and you can use the UI to manage settings etc. Just bear in mind that if this service is redeployed any changes in the UI won't be persisted. If you wish to add new repositories or alter existing ones, use the JSON files within the `./nexus_repos_config` directory. diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index ef1a8dd0f8..3f589488f8 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -35,7 +35,7 @@ parameters: default: false - name: domain_prefix type: string - description: "The FQDN prefix (which will be prepended to {TRE_ID}.{LOCATION}.cloudapp.azure.com) to generate a certificate for" + description: "The FQDN prefix (prepended to {TRE_ID}.{LOCATION}.cloudapp.azure.com) to generate certificate for" - name: cert_name type: string description: "What to call the certificate exported to KeyVault (alphanumeric and '-' only)" From 5d818829f9920387425d16078c4fd88d60a93cee Mon Sep 17 00:00:00 2001 From: Ross Smith Date: Tue, 3 May 2022 15:04:51 +0100 Subject: [PATCH 096/142] Update docs/tre-developers/letsencrypt.md --- docs/tre-developers/letsencrypt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-developers/letsencrypt.md b/docs/tre-developers/letsencrypt.md index 5fdd4da911..0068ec5d0c 100644 --- a/docs/tre-developers/letsencrypt.md +++ b/docs/tre-developers/letsencrypt.md @@ -4,7 +4,7 @@ Certain components of the TRE require the aquisition of a certificate via Letsen In order to aquire these certificates, there must be a public facing endpoint which can be reached by Letsencrypt. -As TREs are secured environments with very few publicly facing points, additional resources are required ensure the certificate can be provisioned for the correct domain. +As TREs are secured environments with very few publicly facing points, additional resources are required to ensure the certificate can be provisioned for the correct domain. The additional resources are as followed: From 7f97c08f75a264da1c39a5add08ff59a3e0d9034 Mon Sep 17 00:00:00 2001 From: Ross Smith Date: Tue, 3 May 2022 15:05:06 +0100 Subject: [PATCH 097/142] Update docs/tre-admins/setup-instructions/configuring-shared-services.md --- .../setup-instructions/configuring-shared-services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 97a64d6741..d9fb0777d7 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -4,7 +4,7 @@ Complete the configuration of the shared services (Nexus and Gitea) from inside ## Configure Nexus -Before deploying the Nexus shared service, you need to make sure that it will have access to a certificate to configure serving proxies over https with. By default, the Nexus service will serve proxies from `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`, and thus it requires a certificate that validates ownership of this domain to use for SSL. +Before deploying the Nexus shared service, you need to make sure that it will have access to a certificate to configure serving secure proxies. By default, the Nexus service will serve proxies from `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`, and thus it requires a certificate that validates ownership of this domain to use for SSL. You can use the Certs Shared Service to set one up by following these steps: From c51cec4f8dbd420c366ee72d069dca28e8215bd6 Mon Sep 17 00:00:00 2001 From: Ross Smith Date: Tue, 3 May 2022 15:07:39 +0100 Subject: [PATCH 098/142] Update docs/tre-developers/letsencrypt.md --- docs/tre-developers/letsencrypt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-developers/letsencrypt.md b/docs/tre-developers/letsencrypt.md index 0068ec5d0c..f117b634e0 100644 --- a/docs/tre-developers/letsencrypt.md +++ b/docs/tre-developers/letsencrypt.md @@ -8,7 +8,7 @@ As TREs are secured environments with very few publicly facing points, additiona The additional resources are as followed: -1. Public IP provisioned in the same location as the web app the certificate is intended for with a domain lable which matches the web apps name. +1. Public IP provisioned in the same location as the web app that the certificate is intended for; this will also have a domain label which matches the web app name. 1. Storage Account with a static web app. 1. Application gateway to route traffic from thepPublic IP to the static web app From a367bb5c6011eb9e63749725b0be41798699309f Mon Sep 17 00:00:00 2001 From: Ross Smith Date: Tue, 3 May 2022 15:08:09 +0100 Subject: [PATCH 099/142] Update docs/tre-developers/letsencrypt.md --- docs/tre-developers/letsencrypt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tre-developers/letsencrypt.md b/docs/tre-developers/letsencrypt.md index f117b634e0..1949202aca 100644 --- a/docs/tre-developers/letsencrypt.md +++ b/docs/tre-developers/letsencrypt.md @@ -10,7 +10,7 @@ The additional resources are as followed: 1. Public IP provisioned in the same location as the web app that the certificate is intended for; this will also have a domain label which matches the web app name. 1. Storage Account with a static web app. -1. Application gateway to route traffic from thepPublic IP to the static web app +1. Application gateway to route traffic from the Public IP to the static web app The following diagram illustrated the flow of data between the resources: From ba8ad037ea5d5991d9337a71c31bb6701288f4a7 Mon Sep 17 00:00:00 2001 From: Ross Smith Date: Thu, 5 May 2022 16:35:34 +0100 Subject: [PATCH 100/142] Update docs/tre-admins/setup-instructions/configuring-shared-services.md Co-authored-by: Marcus Robinson --- .../setup-instructions/configuring-shared-services.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index d9fb0777d7..9f7936eab4 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -43,9 +43,9 @@ After verifying the certificate has been generated, you can deploy Nexus: 1. Run the below commands in your terminal to build, publish and register the Nexus shared service bundle: ```cmd - make bundle-build DIR=./templates/shared_services/certs - make bundle-publish DIR=./templates/shared_services/certs - make bundle-register DIR=./templates/shared_services/certs BUNDLE_TYPE=shared_service + make bundle-build DIR=./templates/shared_services/sonatype-nexus + make bundle-publish DIR=./templates/shared_services/sonatype-nexus + make bundle-register DIR=./templates/shared_services/sonatype-nexus BUNDLE_TYPE=shared_service ``` 1. Navigate to the Swagger UI for your TRE API at `https:///api/docs`, and authenticate if you haven't already by clicking `Authorize`. From 37032bcc163b2e76f331eb055f685430188125ba Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 08:56:09 +0000 Subject: [PATCH 101/142] Fix firewall conflict --- templates/shared_services/certs/terraform/firewall.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/certs/terraform/firewall.tf b/templates/shared_services/certs/terraform/firewall.tf index d7b4b60930..b6a30f0023 100644 --- a/templates/shared_services/certs/terraform/firewall.tf +++ b/templates/shared_services/certs/terraform/firewall.tf @@ -2,7 +2,7 @@ resource "azurerm_firewall_application_rule_collection" "resource_processor_lets name = "resource_processor_subnet_letsencrypt" azure_firewall_name = data.azurerm_firewall.fw.name resource_group_name = data.azurerm_firewall.fw.resource_group_name - priority = 106 + priority = 601 action = "Allow" rule { From 923af133907b39d8588bcc4894b8143db8d25a35 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 09:17:12 +0000 Subject: [PATCH 102/142] Added note to docs for cert kv conflicts --- .../setup-instructions/configuring-shared-services.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 9f7936eab4..772b8d510f 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -32,11 +32,14 @@ You can use the Certs Shared Service to set one up by following these steps: } ``` +!!! caution + If you have KeyVault Purge Protection enabled and are re-deploying your environment using the same `cert_name`, you may encounter this: `Status=409 Code=\"Conflict\" Message=\"Certificate nexus-ssl is currently in a deleted but recoverable state`. You need to either manually recover the certificate or purge it before redeploying; or alternatively give it a new unique name. + 1. Once the shared service has been deployed (which you can check by querying the `/api/shared-services/operations` method), copy its `resource_id`, then find the `POST` operation for `/api/shared-services/{shared_service_id}/invoke_action`, click `Try it out` and paste in the resource id into the `shared_service_id` field, and enter `generate` into the `action` field, then click `Execute`. This will invoke the certs service to use Letsencrypt to generate a certificate for the specified domain prefix followed by `-{TRE_ID}.{LOCATION}.cloudapp.azure.com`, so in our case, having entered `nexus`, this will be `nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`, which will be the public domain for our Nexus service. -Once this has completed, you can verify its success either from the operation output, or by navigating to your core keyvault (`kv-{TRE_ID}`) and looking for a certificate called `nexus-ssl`. +Once this has completed, you can verify its success either from the operation output, or by navigating to your core keyvault (`kv-{TRE_ID}`) and looking for a certificate called `nexus-ssl` (or whatever you called it). After verifying the certificate has been generated, you can deploy Nexus: @@ -57,11 +60,15 @@ After verifying the certificate has been generated, you can deploy Nexus: "templateName": "tre-shared-service-nexus", "properties": { "display_name": "Nexus", - "description": "Proxy public repositories with Nexus" + "description": "Proxy public repositories with Nexus", + "ssl_cert_name": "nexus-ssl" } } ``` +!!! tip + If you called your cert something different in the certs shared service step, make sure that is reflected above. + This will deploy the infrastructure required for Nexus, then start the service and configure it with the repository configurations located in the `./templates/shared_services/sonatype-nexus/scripts/nexus_repos_config` folder. It will also set up HTTPS using the certificate you generated in the previous section, so proxies can be served at `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`. You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/` in the jumpbox and signing in with the username `admin` and the password secret located in your core keyvault, with the key `nexus-admin-password`. Here you should be able to see all of the configured repositories and you can use the UI to manage settings etc. From a03f62db3f0359efe8b34f187ec9b539b52b1120 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 14:14:11 +0000 Subject: [PATCH 103/142] Renamed sonatype-nexus to nexus for new version --- templates/shared_services/{sonatype-nexus => nexus}/.dockerignore | 0 templates/shared_services/{sonatype-nexus => nexus}/.gitignore | 0 .../shared_services/{sonatype-nexus => nexus}/Dockerfile.tmpl | 0 .../shared_services/{sonatype-nexus => nexus}/parameters.json | 0 templates/shared_services/{sonatype-nexus => nexus}/porter.yaml | 0 .../{sonatype-nexus => nexus}/scripts/configure_nexus_repos.sh | 0 .../{sonatype-nexus => nexus}/scripts/configure_nexus_ssl.sh | 0 .../scripts/nexus_repos_config/apt-pypi_proxy_conf.json | 0 .../scripts/nexus_repos_config/conda_forge_proxy_conf.json | 0 .../scripts/nexus_repos_config/conda_proxy_conf.json | 0 .../scripts/nexus_repos_config/docker_gpg_proxy_conf.json | 0 .../scripts/nexus_repos_config/docker_hub_proxy_conf.json | 0 .../scripts/nexus_repos_config/docker_proxy_conf.json | 0 .../scripts/nexus_repos_config/pypi_proxy_conf.json | 0 .../scripts/nexus_repos_config/ubuntu_proxy_conf.json | 0 .../scripts/nexus_repos_config/ubuntu_security_proxy_conf.json | 0 .../{sonatype-nexus => nexus}/scripts/reset_nexus_password.sh | 0 .../{sonatype-nexus => nexus}/template_schema.json | 0 .../{sonatype-nexus => nexus}/terraform/.terraform.lock.hcl | 0 .../{sonatype-nexus => nexus}/terraform/cloud-config.yaml | 0 .../shared_services/{sonatype-nexus => nexus}/terraform/data.tf | 0 .../shared_services/{sonatype-nexus => nexus}/terraform/deploy.sh | 0 .../{sonatype-nexus => nexus}/terraform/destroy.sh | 0 .../{sonatype-nexus => nexus}/terraform/firewall.tf | 0 .../shared_services/{sonatype-nexus => nexus}/terraform/locals.tf | 0 .../shared_services/{sonatype-nexus => nexus}/terraform/main.tf | 0 .../shared_services/{sonatype-nexus => nexus}/terraform/output.tf | 0 .../{sonatype-nexus => nexus}/terraform/variables.tf | 0 .../shared_services/{sonatype-nexus => nexus}/terraform/vm.tf | 0 29 files changed, 0 insertions(+), 0 deletions(-) rename templates/shared_services/{sonatype-nexus => nexus}/.dockerignore (100%) rename templates/shared_services/{sonatype-nexus => nexus}/.gitignore (100%) rename templates/shared_services/{sonatype-nexus => nexus}/Dockerfile.tmpl (100%) rename templates/shared_services/{sonatype-nexus => nexus}/parameters.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/porter.yaml (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/configure_nexus_repos.sh (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/configure_nexus_ssl.sh (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/apt-pypi_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/conda_forge_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/conda_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/docker_gpg_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/docker_hub_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/docker_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/pypi_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/ubuntu_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/scripts/reset_nexus_password.sh (100%) rename templates/shared_services/{sonatype-nexus => nexus}/template_schema.json (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/.terraform.lock.hcl (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/cloud-config.yaml (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/data.tf (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/deploy.sh (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/destroy.sh (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/firewall.tf (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/locals.tf (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/main.tf (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/output.tf (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/variables.tf (100%) rename templates/shared_services/{sonatype-nexus => nexus}/terraform/vm.tf (100%) diff --git a/templates/shared_services/sonatype-nexus/.dockerignore b/templates/shared_services/nexus/.dockerignore similarity index 100% rename from templates/shared_services/sonatype-nexus/.dockerignore rename to templates/shared_services/nexus/.dockerignore diff --git a/templates/shared_services/sonatype-nexus/.gitignore b/templates/shared_services/nexus/.gitignore similarity index 100% rename from templates/shared_services/sonatype-nexus/.gitignore rename to templates/shared_services/nexus/.gitignore diff --git a/templates/shared_services/sonatype-nexus/Dockerfile.tmpl b/templates/shared_services/nexus/Dockerfile.tmpl similarity index 100% rename from templates/shared_services/sonatype-nexus/Dockerfile.tmpl rename to templates/shared_services/nexus/Dockerfile.tmpl diff --git a/templates/shared_services/sonatype-nexus/parameters.json b/templates/shared_services/nexus/parameters.json similarity index 100% rename from templates/shared_services/sonatype-nexus/parameters.json rename to templates/shared_services/nexus/parameters.json diff --git a/templates/shared_services/sonatype-nexus/porter.yaml b/templates/shared_services/nexus/porter.yaml similarity index 100% rename from templates/shared_services/sonatype-nexus/porter.yaml rename to templates/shared_services/nexus/porter.yaml diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh b/templates/shared_services/nexus/scripts/configure_nexus_repos.sh similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/configure_nexus_repos.sh rename to templates/shared_services/nexus/scripts/configure_nexus_repos.sh diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh b/templates/shared_services/nexus/scripts/configure_nexus_ssl.sh similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/configure_nexus_ssl.sh rename to templates/shared_services/nexus/scripts/configure_nexus_ssl.sh diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/apt-pypi_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/apt-pypi_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/apt-pypi_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/apt-pypi_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/conda_forge_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/conda_forge_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/conda_forge_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/conda_forge_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/conda_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/conda_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/conda_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/conda_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_gpg_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/docker_gpg_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_gpg_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/docker_gpg_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_hub_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/docker_hub_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_hub_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/docker_hub_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/docker_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/docker_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/docker_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/pypi_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/pypi_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/pypi_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/pypi_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/ubuntu_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/ubuntu_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/ubuntu_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/ubuntu_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json b/templates/shared_services/nexus/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json rename to templates/shared_services/nexus/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json diff --git a/templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh b/templates/shared_services/nexus/scripts/reset_nexus_password.sh similarity index 100% rename from templates/shared_services/sonatype-nexus/scripts/reset_nexus_password.sh rename to templates/shared_services/nexus/scripts/reset_nexus_password.sh diff --git a/templates/shared_services/sonatype-nexus/template_schema.json b/templates/shared_services/nexus/template_schema.json similarity index 100% rename from templates/shared_services/sonatype-nexus/template_schema.json rename to templates/shared_services/nexus/template_schema.json diff --git a/templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl b/templates/shared_services/nexus/terraform/.terraform.lock.hcl similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl rename to templates/shared_services/nexus/terraform/.terraform.lock.hcl diff --git a/templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml b/templates/shared_services/nexus/terraform/cloud-config.yaml similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/cloud-config.yaml rename to templates/shared_services/nexus/terraform/cloud-config.yaml diff --git a/templates/shared_services/sonatype-nexus/terraform/data.tf b/templates/shared_services/nexus/terraform/data.tf similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/data.tf rename to templates/shared_services/nexus/terraform/data.tf diff --git a/templates/shared_services/sonatype-nexus/terraform/deploy.sh b/templates/shared_services/nexus/terraform/deploy.sh similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/deploy.sh rename to templates/shared_services/nexus/terraform/deploy.sh diff --git a/templates/shared_services/sonatype-nexus/terraform/destroy.sh b/templates/shared_services/nexus/terraform/destroy.sh similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/destroy.sh rename to templates/shared_services/nexus/terraform/destroy.sh diff --git a/templates/shared_services/sonatype-nexus/terraform/firewall.tf b/templates/shared_services/nexus/terraform/firewall.tf similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/firewall.tf rename to templates/shared_services/nexus/terraform/firewall.tf diff --git a/templates/shared_services/sonatype-nexus/terraform/locals.tf b/templates/shared_services/nexus/terraform/locals.tf similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/locals.tf rename to templates/shared_services/nexus/terraform/locals.tf diff --git a/templates/shared_services/sonatype-nexus/terraform/main.tf b/templates/shared_services/nexus/terraform/main.tf similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/main.tf rename to templates/shared_services/nexus/terraform/main.tf diff --git a/templates/shared_services/sonatype-nexus/terraform/output.tf b/templates/shared_services/nexus/terraform/output.tf similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/output.tf rename to templates/shared_services/nexus/terraform/output.tf diff --git a/templates/shared_services/sonatype-nexus/terraform/variables.tf b/templates/shared_services/nexus/terraform/variables.tf similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/variables.tf rename to templates/shared_services/nexus/terraform/variables.tf diff --git a/templates/shared_services/sonatype-nexus/terraform/vm.tf b/templates/shared_services/nexus/terraform/vm.tf similarity index 100% rename from templates/shared_services/sonatype-nexus/terraform/vm.tf rename to templates/shared_services/nexus/terraform/vm.tf From 9e9e392f41b830651e2f26799ceb0cec54a5376e Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 15:22:18 +0100 Subject: [PATCH 104/142] Added old nexus service code --- .../sonatype-nexus/.dockerignore | 6 + .../shared_services/sonatype-nexus/.gitignore | 1 + .../sonatype-nexus/Dockerfile.tmpl | 29 +++ .../sonatype-nexus/nexus.properties | 1 + .../sonatype-nexus/parameters.json | 32 +++ .../sonatype-nexus/porter.yaml | 68 ++++++ .../sonatype-nexus/scripts/configure_nexus.sh | 87 +++++++ .../nexus_config/apt-pypi_proxy_conf.json | 36 +++ .../nexus_config/conda_forge_proxy_conf.json | 32 +++ .../nexus_config/conda_proxy_conf.json | 32 +++ .../nexus_config/docker_gpg_proxy_conf.json | 32 +++ .../nexus_config/docker_hub_proxy_conf.json | 40 ++++ .../nexus_config/docker_proxy_conf.json | 36 +++ .../scripts/nexus_config/pypi_proxy_conf.json | 32 +++ .../nexus_config/ubuntu_proxy_conf.json | 36 +++ .../ubuntu_security_proxy_conf.json | 36 +++ .../sonatype-nexus/template_schema.json | 9 + .../terraform/.terraform.lock.hcl | 39 ++++ .../sonatype-nexus/terraform/data.tf | 50 ++++ .../sonatype-nexus/terraform/deploy.sh | 9 + .../sonatype-nexus/terraform/destroy.sh | 9 + .../sonatype-nexus/terraform/firewall.tf | 22 ++ .../sonatype-nexus/terraform/locals.tf | 7 + .../sonatype-nexus/terraform/main.tf | 15 ++ .../sonatype-nexus/terraform/output.tf | 3 + .../sonatype-nexus/terraform/variables.tf | 22 ++ .../sonatype-nexus/terraform/webapp.tf | 213 ++++++++++++++++++ 27 files changed, 934 insertions(+) create mode 100644 templates/shared_services/sonatype-nexus/.dockerignore create mode 100644 templates/shared_services/sonatype-nexus/.gitignore create mode 100644 templates/shared_services/sonatype-nexus/Dockerfile.tmpl create mode 100644 templates/shared_services/sonatype-nexus/nexus.properties create mode 100755 templates/shared_services/sonatype-nexus/parameters.json create mode 100644 templates/shared_services/sonatype-nexus/porter.yaml create mode 100644 templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/apt-pypi_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/conda_forge_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/conda_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_gpg_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_hub_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/docker_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/pypi_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/ubuntu_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/scripts/nexus_config/ubuntu_security_proxy_conf.json create mode 100644 templates/shared_services/sonatype-nexus/template_schema.json create mode 100644 templates/shared_services/sonatype-nexus/terraform/.terraform.lock.hcl create mode 100644 templates/shared_services/sonatype-nexus/terraform/data.tf create mode 100755 templates/shared_services/sonatype-nexus/terraform/deploy.sh create mode 100755 templates/shared_services/sonatype-nexus/terraform/destroy.sh create mode 100644 templates/shared_services/sonatype-nexus/terraform/firewall.tf create mode 100644 templates/shared_services/sonatype-nexus/terraform/locals.tf create mode 100644 templates/shared_services/sonatype-nexus/terraform/main.tf create mode 100644 templates/shared_services/sonatype-nexus/terraform/output.tf create mode 100644 templates/shared_services/sonatype-nexus/terraform/variables.tf create mode 100644 templates/shared_services/sonatype-nexus/terraform/webapp.tf diff --git a/templates/shared_services/sonatype-nexus/.dockerignore b/templates/shared_services/sonatype-nexus/.dockerignore new file mode 100644 index 0000000000..36177107fc --- /dev/null +++ b/templates/shared_services/sonatype-nexus/.dockerignore @@ -0,0 +1,6 @@ +# See https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Put files here that you don't want copied into your bundle's invocation image +.gitignore +**/.terraform/* +**/*_backend.tf +Dockerfile.tmpl diff --git a/templates/shared_services/sonatype-nexus/.gitignore b/templates/shared_services/sonatype-nexus/.gitignore new file mode 100644 index 0000000000..e08a3e22b9 --- /dev/null +++ b/templates/shared_services/sonatype-nexus/.gitignore @@ -0,0 +1 @@ +.cnab/ diff --git a/templates/shared_services/sonatype-nexus/Dockerfile.tmpl b/templates/shared_services/sonatype-nexus/Dockerfile.tmpl new file mode 100644 index 0000000000..3f8524ce64 --- /dev/null +++ b/templates/shared_services/sonatype-nexus/Dockerfile.tmpl @@ -0,0 +1,29 @@ +FROM debian:stretch-slim + +ARG BUNDLE_DIR + +RUN apt-get update && apt-get install -y ca-certificates + +# Install Azure CLI +RUN apt-get update \ + && apt-get install -y ca-certificates jq curl apt-transport-https lsb-release gnupg \ + && curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null \ + && AZ_REPO=$(lsb_release -cs) \ + && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list \ + && apt-get update && apt-get -y install azure-cli + +# This is a template Dockerfile for the bundle's invocation image +# You can customize it to use different base images, install tools and copy configuration files. +# +# Porter will use it as a template and append lines to it for the mixins +# and to set the CMD appropriately for the CNAB specification. +# +# Add the following line to porter.yaml to instruct Porter to use this template +# dockerfile: Dockerfile.tmpl + +# You can control where the mixin's Dockerfile lines are inserted into this file by moving "# PORTER_MIXINS" line +# another location in this file. If you remove that line, the mixins generated content is appended to this file. +# PORTER_MIXINS + +# Use the BUNDLE_DIR build argument to copy files into the bundle +COPY . $BUNDLE_DIR diff --git a/templates/shared_services/sonatype-nexus/nexus.properties b/templates/shared_services/sonatype-nexus/nexus.properties new file mode 100644 index 0000000000..2ae16dd047 --- /dev/null +++ b/templates/shared_services/sonatype-nexus/nexus.properties @@ -0,0 +1 @@ +nexus.skipDefaultRepositories=true diff --git a/templates/shared_services/sonatype-nexus/parameters.json b/templates/shared_services/sonatype-nexus/parameters.json new file mode 100755 index 0000000000..c01ad0266b --- /dev/null +++ b/templates/shared_services/sonatype-nexus/parameters.json @@ -0,0 +1,32 @@ +{ + "schemaVersion": "1.0.0-DRAFT+TODO", + "name": "base", + "created": "2021-06-04T13:37:29.5071039+03:00", + "modified": "2021-06-04T13:37:29.5071039+03:00", + "parameters": [ + { + "name": "tre_id", + "source": { + "env": "TRE_ID" + } + }, + { + "name": "tfstate_container_name", + "source": { + "env": "TERRAFORM_STATE_CONTAINER_NAME" + } + }, + { + "name": "tfstate_resource_group_name", + "source": { + "env": "MGMT_RESOURCE_GROUP_NAME" + } + }, + { + "name": "tfstate_storage_account_name", + "source": { + "env": "MGMT_STORAGE_ACCOUNT_NAME" + } + } + ] +} diff --git a/templates/shared_services/sonatype-nexus/porter.yaml b/templates/shared_services/sonatype-nexus/porter.yaml new file mode 100644 index 0000000000..150565da27 --- /dev/null +++ b/templates/shared_services/sonatype-nexus/porter.yaml @@ -0,0 +1,68 @@ +--- +name: tre-shared-service-nexus +version: 0.0.2 +description: "A Sonatype Nexus shared service" +registry: azuretre +credentials: + - name: azure_tenant_id + env: ARM_TENANT_ID + - name: azure_subscription_id + env: ARM_SUBSCRIPTION_ID + - name: azure_client_id + env: ARM_CLIENT_ID + - name: azure_client_secret + env: ARM_CLIENT_SECRET +parameters: + - name: tre_id + type: string + description: "The ID of the parent TRE instance e.g., mytre-dev-3142" + - name: tfstate_resource_group_name + type: string + description: "Resource group containing the Terraform state storage account" + - name: tfstate_storage_account_name + type: string + description: "The name of the Terraform state storage account" + - name: tfstate_container_name + type: string + default: "tfstate" + description: "The name of the Terraform state storage container" + - name: arm_use_msi + env: ARM_USE_MSI + default: false +mixins: + - exec + - az + - terraform: + clientVersion: 1.1.5 +install: + - terraform: + description: "Deploy shared service" + input: false + vars: + tre_id: "{{ bundle.parameters.tre_id }}" + backendConfig: + resource_group_name: + "{{ bundle.parameters.tfstate_resource_group_name }}" + storage_account_name: + "{{ bundle.parameters.tfstate_storage_account_name }}" + container_name: "{{ bundle.parameters.tfstate_container_name }}" + key: "{{ bundle.parameters.tre_id }}-shared-service-sonatype-nexus" +upgrade: + - exec: + description: "Upgrade shared service" + command: echo + arguments: + - "This shared service does not implement upgrade action" +uninstall: + - terraform: + description: "Tear down shared service" + input: false + vars: + tre_id: "{{ bundle.parameters.tre_id }}" + backendConfig: + resource_group_name: + "{{ bundle.parameters.tfstate_resource_group_name }}" + storage_account_name: + "{{ bundle.parameters.tfstate_storage_account_name }}" + container_name: "{{ bundle.parameters.tfstate_container_name }}" + key: "{{ bundle.parameters.tre_id }}-shared-service-sonatype-nexus" diff --git a/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh new file mode 100644 index 0000000000..25909da9bd --- /dev/null +++ b/templates/shared_services/sonatype-nexus/scripts/configure_nexus.sh @@ -0,0 +1,87 @@ +#!/bin/bash +set -e + +function usage() { + cat < Date: Thu, 12 May 2022 15:45:43 +0100 Subject: [PATCH 105/142] Lint fix --- templates/shared_services/nexus/terraform/deploy.sh | 1 + templates/shared_services/nexus/terraform/destroy.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/templates/shared_services/nexus/terraform/deploy.sh b/templates/shared_services/nexus/terraform/deploy.sh index 14160537eb..007f4acc3c 100755 --- a/templates/shared_services/nexus/terraform/deploy.sh +++ b/templates/shared_services/nexus/terraform/deploy.sh @@ -1,3 +1,4 @@ +#!/bin/bash export TF_LOG="" terraform init -input=false -backend=true -reconfigure \ diff --git a/templates/shared_services/nexus/terraform/destroy.sh b/templates/shared_services/nexus/terraform/destroy.sh index 14ad9ec212..f93b3830b3 100755 --- a/templates/shared_services/nexus/terraform/destroy.sh +++ b/templates/shared_services/nexus/terraform/destroy.sh @@ -1,3 +1,4 @@ +#!/bin/bash export TF_LOG="" terraform init -input=false -backend=true -reconfigure \ From 9def053c9ff184298c422dbac98453a8d4311c98 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 16:08:18 +0100 Subject: [PATCH 106/142] Renamed folder to be obvious as the nexus-vm --- .../{nexus => sonatype-nexus-vm}/.dockerignore | 0 .../shared_services/{nexus => sonatype-nexus-vm}/.gitignore | 0 .../{nexus => sonatype-nexus-vm}/Dockerfile.tmpl | 0 .../{nexus => sonatype-nexus-vm}/parameters.json | 0 .../shared_services/{nexus => sonatype-nexus-vm}/porter.yaml | 4 ++-- .../scripts/configure_nexus_repos.sh | 0 .../scripts/configure_nexus_ssl.sh | 0 .../scripts/nexus_repos_config/apt-pypi_proxy_conf.json | 0 .../scripts/nexus_repos_config/conda_forge_proxy_conf.json | 0 .../scripts/nexus_repos_config/conda_proxy_conf.json | 0 .../scripts/nexus_repos_config/docker_gpg_proxy_conf.json | 0 .../scripts/nexus_repos_config/docker_hub_proxy_conf.json | 0 .../scripts/nexus_repos_config/docker_proxy_conf.json | 0 .../scripts/nexus_repos_config/pypi_proxy_conf.json | 0 .../scripts/nexus_repos_config/ubuntu_proxy_conf.json | 0 .../nexus_repos_config/ubuntu_security_proxy_conf.json | 0 .../scripts/reset_nexus_password.sh | 0 .../{nexus => sonatype-nexus-vm}/template_schema.json | 0 .../terraform/.terraform.lock.hcl | 0 .../{nexus => sonatype-nexus-vm}/terraform/cloud-config.yaml | 0 .../{nexus => sonatype-nexus-vm}/terraform/data.tf | 0 .../{nexus => sonatype-nexus-vm}/terraform/deploy.sh | 0 .../{nexus => sonatype-nexus-vm}/terraform/destroy.sh | 0 .../{nexus => sonatype-nexus-vm}/terraform/firewall.tf | 0 .../{nexus => sonatype-nexus-vm}/terraform/locals.tf | 0 .../{nexus => sonatype-nexus-vm}/terraform/main.tf | 0 .../{nexus => sonatype-nexus-vm}/terraform/output.tf | 0 .../{nexus => sonatype-nexus-vm}/terraform/variables.tf | 0 .../{nexus => sonatype-nexus-vm}/terraform/vm.tf | 0 29 files changed, 2 insertions(+), 2 deletions(-) rename templates/shared_services/{nexus => sonatype-nexus-vm}/.dockerignore (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/.gitignore (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/Dockerfile.tmpl (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/parameters.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/porter.yaml (97%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/configure_nexus_repos.sh (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/configure_nexus_ssl.sh (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/apt-pypi_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/conda_forge_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/conda_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/docker_gpg_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/docker_hub_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/docker_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/pypi_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/ubuntu_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/scripts/reset_nexus_password.sh (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/template_schema.json (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/.terraform.lock.hcl (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/cloud-config.yaml (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/data.tf (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/deploy.sh (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/destroy.sh (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/firewall.tf (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/locals.tf (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/main.tf (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/output.tf (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/variables.tf (100%) rename templates/shared_services/{nexus => sonatype-nexus-vm}/terraform/vm.tf (100%) diff --git a/templates/shared_services/nexus/.dockerignore b/templates/shared_services/sonatype-nexus-vm/.dockerignore similarity index 100% rename from templates/shared_services/nexus/.dockerignore rename to templates/shared_services/sonatype-nexus-vm/.dockerignore diff --git a/templates/shared_services/nexus/.gitignore b/templates/shared_services/sonatype-nexus-vm/.gitignore similarity index 100% rename from templates/shared_services/nexus/.gitignore rename to templates/shared_services/sonatype-nexus-vm/.gitignore diff --git a/templates/shared_services/nexus/Dockerfile.tmpl b/templates/shared_services/sonatype-nexus-vm/Dockerfile.tmpl similarity index 100% rename from templates/shared_services/nexus/Dockerfile.tmpl rename to templates/shared_services/sonatype-nexus-vm/Dockerfile.tmpl diff --git a/templates/shared_services/nexus/parameters.json b/templates/shared_services/sonatype-nexus-vm/parameters.json similarity index 100% rename from templates/shared_services/nexus/parameters.json rename to templates/shared_services/sonatype-nexus-vm/parameters.json diff --git a/templates/shared_services/nexus/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml similarity index 97% rename from templates/shared_services/nexus/porter.yaml rename to templates/shared_services/sonatype-nexus-vm/porter.yaml index b42c264a2b..cb15837b36 100644 --- a/templates/shared_services/nexus/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -1,6 +1,6 @@ --- -name: tre-shared-service-nexus -version: 0.0.3 +name: tre-shared-service-sonatype-nexus +version: 1.0.1 description: "A Sonatype Nexus shared service" registry: azuretre credentials: diff --git a/templates/shared_services/nexus/scripts/configure_nexus_repos.sh b/templates/shared_services/sonatype-nexus-vm/scripts/configure_nexus_repos.sh similarity index 100% rename from templates/shared_services/nexus/scripts/configure_nexus_repos.sh rename to templates/shared_services/sonatype-nexus-vm/scripts/configure_nexus_repos.sh diff --git a/templates/shared_services/nexus/scripts/configure_nexus_ssl.sh b/templates/shared_services/sonatype-nexus-vm/scripts/configure_nexus_ssl.sh similarity index 100% rename from templates/shared_services/nexus/scripts/configure_nexus_ssl.sh rename to templates/shared_services/sonatype-nexus-vm/scripts/configure_nexus_ssl.sh diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/apt-pypi_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/apt-pypi_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/apt-pypi_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/apt-pypi_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/conda_forge_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/conda_forge_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/conda_forge_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/conda_forge_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/conda_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/conda_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/conda_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/conda_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/docker_gpg_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/docker_gpg_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/docker_gpg_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/docker_gpg_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/docker_hub_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/docker_hub_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/docker_hub_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/docker_hub_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/docker_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/docker_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/docker_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/docker_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/pypi_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/pypi_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/pypi_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/pypi_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/ubuntu_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/ubuntu_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/ubuntu_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/ubuntu_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json similarity index 100% rename from templates/shared_services/nexus/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json rename to templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/ubuntu_security_proxy_conf.json diff --git a/templates/shared_services/nexus/scripts/reset_nexus_password.sh b/templates/shared_services/sonatype-nexus-vm/scripts/reset_nexus_password.sh similarity index 100% rename from templates/shared_services/nexus/scripts/reset_nexus_password.sh rename to templates/shared_services/sonatype-nexus-vm/scripts/reset_nexus_password.sh diff --git a/templates/shared_services/nexus/template_schema.json b/templates/shared_services/sonatype-nexus-vm/template_schema.json similarity index 100% rename from templates/shared_services/nexus/template_schema.json rename to templates/shared_services/sonatype-nexus-vm/template_schema.json diff --git a/templates/shared_services/nexus/terraform/.terraform.lock.hcl b/templates/shared_services/sonatype-nexus-vm/terraform/.terraform.lock.hcl similarity index 100% rename from templates/shared_services/nexus/terraform/.terraform.lock.hcl rename to templates/shared_services/sonatype-nexus-vm/terraform/.terraform.lock.hcl diff --git a/templates/shared_services/nexus/terraform/cloud-config.yaml b/templates/shared_services/sonatype-nexus-vm/terraform/cloud-config.yaml similarity index 100% rename from templates/shared_services/nexus/terraform/cloud-config.yaml rename to templates/shared_services/sonatype-nexus-vm/terraform/cloud-config.yaml diff --git a/templates/shared_services/nexus/terraform/data.tf b/templates/shared_services/sonatype-nexus-vm/terraform/data.tf similarity index 100% rename from templates/shared_services/nexus/terraform/data.tf rename to templates/shared_services/sonatype-nexus-vm/terraform/data.tf diff --git a/templates/shared_services/nexus/terraform/deploy.sh b/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh similarity index 100% rename from templates/shared_services/nexus/terraform/deploy.sh rename to templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh diff --git a/templates/shared_services/nexus/terraform/destroy.sh b/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh similarity index 100% rename from templates/shared_services/nexus/terraform/destroy.sh rename to templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh diff --git a/templates/shared_services/nexus/terraform/firewall.tf b/templates/shared_services/sonatype-nexus-vm/terraform/firewall.tf similarity index 100% rename from templates/shared_services/nexus/terraform/firewall.tf rename to templates/shared_services/sonatype-nexus-vm/terraform/firewall.tf diff --git a/templates/shared_services/nexus/terraform/locals.tf b/templates/shared_services/sonatype-nexus-vm/terraform/locals.tf similarity index 100% rename from templates/shared_services/nexus/terraform/locals.tf rename to templates/shared_services/sonatype-nexus-vm/terraform/locals.tf diff --git a/templates/shared_services/nexus/terraform/main.tf b/templates/shared_services/sonatype-nexus-vm/terraform/main.tf similarity index 100% rename from templates/shared_services/nexus/terraform/main.tf rename to templates/shared_services/sonatype-nexus-vm/terraform/main.tf diff --git a/templates/shared_services/nexus/terraform/output.tf b/templates/shared_services/sonatype-nexus-vm/terraform/output.tf similarity index 100% rename from templates/shared_services/nexus/terraform/output.tf rename to templates/shared_services/sonatype-nexus-vm/terraform/output.tf diff --git a/templates/shared_services/nexus/terraform/variables.tf b/templates/shared_services/sonatype-nexus-vm/terraform/variables.tf similarity index 100% rename from templates/shared_services/nexus/terraform/variables.tf rename to templates/shared_services/sonatype-nexus-vm/terraform/variables.tf diff --git a/templates/shared_services/nexus/terraform/vm.tf b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf similarity index 100% rename from templates/shared_services/nexus/terraform/vm.tf rename to templates/shared_services/sonatype-nexus-vm/terraform/vm.tf From 57cab777ef40c6a97324456f38787a5041a4d863 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 17:13:30 +0100 Subject: [PATCH 107/142] Added docs for upgrade path --- .../configuring-shared-services.md | 43 +++++++++++++------ .../guacamole-azure-linuxvm/porter.yaml | 2 +- .../guacamole-azure-windowsvm/porter.yaml | 2 +- .../guacamole-dev-vm/porter.yaml | 2 +- .../guacamole-dev-vm/terraform/locals.tf | 2 +- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 772b8d510f..1ca845c36e 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -1,8 +1,17 @@ # Configuring Shared Services -Complete the configuration of the shared services (Nexus and Gitea) from inside of the TRE environment. +## Deploy/configure Nexus -## Configure Nexus +There is a new Nexus shared service which can be located in the `./templates/shared_services/sonatype-nexus-vm` directory, with the bundle name `tre-shared-service-sonatype-nexus`, which is now hosted using a VM to enable additional configuration required for proxying certain repositories. This has been created as a separate service as the domain name exposed for proxies will be different to the one used by the original Nexus service and thus will break any user resources configured with the old proxy URL. + +The original Nexus service that runs on App Service (located in `./templates/shared_services/sonatype-nexus`) has the bundle name `tre-shared-service-nexus` so can co-exist with the new VM-based shared service to enable smoother upgrading of existing resources. + +If you're deploying a brand new environment you should deploy the new service (read section `A`). If you wish to migrate from an existing App Service Nexus service to the new VM service, first deploy the new service (section `A`) then proceed to section `B`. + +!!! info + The Makefile commands for deploying shared services temporarily target the old Nexus service so that existing environments won't have a new Nexus service deployed automatically by CICD and introduce breaking changes. The new Nexus service will need to be deployed manually using the steps detailed below. + +### A. Deploy & configure new Nexus service (hosted on VM) Before deploying the Nexus shared service, you need to make sure that it will have access to a certificate to configure serving secure proxies. By default, the Nexus service will serve proxies from `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`, and thus it requires a certificate that validates ownership of this domain to use for SSL. @@ -16,9 +25,9 @@ You can use the Certs Shared Service to set one up by following these steps: make bundle-register DIR=./templates/shared_services/certs BUNDLE_TYPE=shared_service ``` -1. Navigate to the Swagger UI for your TRE API at `https:///api/docs`, and authenticate if you haven't already by clicking `Authorize`. +2. Navigate to the Swagger UI for your TRE API at `https:///api/docs`, and authenticate if you haven't already by clicking `Authorize`. -1. Click `Try it out` on the `POST` `/api/shared-services` operation, and paste the following to deploy the certs service: +3. Click `Try it out` on the `POST` `/api/shared-services` operation, and paste the following to deploy the certs service: ```json { @@ -33,9 +42,9 @@ You can use the Certs Shared Service to set one up by following these steps: ``` !!! caution - If you have KeyVault Purge Protection enabled and are re-deploying your environment using the same `cert_name`, you may encounter this: `Status=409 Code=\"Conflict\" Message=\"Certificate nexus-ssl is currently in a deleted but recoverable state`. You need to either manually recover the certificate or purge it before redeploying; or alternatively give it a new unique name. + If you have KeyVault Purge Protection enabled and are re-deploying your environment using the same `cert_name`, you may encounter this: `Status=409 Code=\"Conflict\" Message=\"Certificate nexus-ssl is currently in a deleted but recoverable state`. You need to either manually recover the certificate or purge it before redeploying; or alternatively give it a new unique name. -1. Once the shared service has been deployed (which you can check by querying the `/api/shared-services/operations` method), copy its `resource_id`, then find the `POST` operation for `/api/shared-services/{shared_service_id}/invoke_action`, click `Try it out` and paste in the resource id into the `shared_service_id` field, and enter `generate` into the `action` field, then click `Execute`. +4. Once the shared service has been deployed (which you can check by querying the `/api/shared-services/operations` method), copy its `resource_id`, then find the `POST` operation for `/api/shared-services/{shared_service_id}/invoke_action`, click `Try it out` and paste in the resource id into the `shared_service_id` field, and enter `generate` into the `action` field, then click `Execute`. This will invoke the certs service to use Letsencrypt to generate a certificate for the specified domain prefix followed by `-{TRE_ID}.{LOCATION}.cloudapp.azure.com`, so in our case, having entered `nexus`, this will be `nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`, which will be the public domain for our Nexus service. @@ -46,9 +55,9 @@ After verifying the certificate has been generated, you can deploy Nexus: 1. Run the below commands in your terminal to build, publish and register the Nexus shared service bundle: ```cmd - make bundle-build DIR=./templates/shared_services/sonatype-nexus - make bundle-publish DIR=./templates/shared_services/sonatype-nexus - make bundle-register DIR=./templates/shared_services/sonatype-nexus BUNDLE_TYPE=shared_service + make bundle-build DIR=./templates/shared_services/sonatype-nexus-vm + make bundle-publish DIR=./templates/shared_services/sonatype-nexus-vm + make bundle-register DIR=./templates/shared_services/sonatype-nexus-vm BUNDLE_TYPE=shared_service ``` 1. Navigate to the Swagger UI for your TRE API at `https:///api/docs`, and authenticate if you haven't already by clicking `Authorize`. @@ -57,7 +66,7 @@ After verifying the certificate has been generated, you can deploy Nexus: ```json { - "templateName": "tre-shared-service-nexus", + "templateName": "tre-shared-service-sonatype-nexus", "properties": { "display_name": "Nexus", "description": "Proxy public repositories with Nexus", @@ -67,14 +76,24 @@ After verifying the certificate has been generated, you can deploy Nexus: ``` !!! tip - If you called your cert something different in the certs shared service step, make sure that is reflected above. + If you called your cert something different in the certs shared service step, make sure that is reflected above. -This will deploy the infrastructure required for Nexus, then start the service and configure it with the repository configurations located in the `./templates/shared_services/sonatype-nexus/scripts/nexus_repos_config` folder. It will also set up HTTPS using the certificate you generated in the previous section, so proxies can be served at `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`. +This will deploy the infrastructure required for Nexus, then start the service and configure it with the repository configurations located in the `./templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config` folder. It will also set up HTTPS using the certificate you generated in the previous section, so proxies can be served at `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`. You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/` in the jumpbox and signing in with the username `admin` and the password secret located in your core keyvault, with the key `nexus-admin-password`. Here you should be able to see all of the configured repositories and you can use the UI to manage settings etc. Just bear in mind that if this service is redeployed any changes in the UI won't be persisted. If you wish to add new repositories or alter existing ones, use the JSON files within the `./nexus_repos_config` directory. +### B. Migrate from existing Nexus service (hosted on App Service) + +Once you've created the new VM-based Nexus service by following section `A`, you can migrate from the old App Service Nexus service by following these steps: + +1. Identify any existing Guacamole user resources that are using the old proxy URL (`https://nexus-{TRE_ID}.azurewebsites.net/`). These will be any VMs with bundle versions < `0.2.0`. + +1. These will need to be either **re-deployed** with the new template versions `0.2.0` or later (which target the new proxy URL format of `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`), or manually have their proxy URLs updated by remoting into the VMs and updating the various configuration files of required package managers with the new URL. For example, pip will need the `index`, `index-url` and `trusted-host` values in the global `pip.conf` file to be modified to use the new URL. + +1. Once you've confirmed there are no dependencies on the old Nexus shared service, you can delete it using the API. + ## Configure Gitea repositories Note : This is a Gitea *shared service* which will be accessible from all workspaces intended for mirroring external Git repositories. A Gitea *workspace service* can also be deployed per workspace to enable Gitea to be used within a specific workspace. diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index 8c5874b840..0610d8ffe5 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-linuxvm -version: 0.1.11 +version: 0.2.0 description: "An Azure TRE User Resource Template for Guacamole (Linux)" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index 149b438516..694fd1e046 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-windowsvm -version: 0.1.12 +version: 0.2.0 description: "An Azure TRE User Resource Template for Guacamole (Windows 10)" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/porter.yaml index 75ee52e3ad..db34241cdf 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-dev-vm -version: 0.1.11 +version: 0.2.0 description: "An Azure TRE User Resource Template for a Dev VM" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/locals.tf index e684eb37d5..4846e924d7 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/locals.tf @@ -9,7 +9,7 @@ locals { vm_name = "linuxvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") storage_name = lower(replace("stg${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", "")) - nexus_proxy_url = "https://nexus-${var.tre_id}.azurewebsites.net" + nexus_proxy_url = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" vm_size = { "2 CPU | 8GB RAM" = { value = "Standard_D2s_v5" }, "4 CPU | 16GB RAM" = { value = "Standard_D4s_v5" }, From a1c598cf40c27c24fde2e32d44bed065c9c8842a Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 17:25:26 +0100 Subject: [PATCH 108/142] Added data.azurerm rg core --- .../guacamole-azure-linuxvm/terraform/locals.tf | 4 ---- .../user_resources/guacamole-azure-linuxvm/terraform/main.tf | 4 ++++ .../guacamole-azure-windowsvm/terraform/locals.tf | 4 ---- .../guacamole-azure-windowsvm/terraform/main.tf | 4 ++++ .../user_resources/guacamole-dev-vm/terraform/main.tf | 4 ++++ 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf index 97ab00acfe..a2d35fef10 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf @@ -1,7 +1,3 @@ -data "azurerm_resource_group" "core" { - name = "rg-${var.tre_id}" -} - locals { short_service_id = substr(var.tre_resource_id, -4, -1) short_workspace_id = substr(var.workspace_id, -4, -1) diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/main.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/main.tf index da10b88845..01b0ce69bb 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/main.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/main.tf @@ -21,6 +21,10 @@ data "azurerm_resource_group" "ws" { name = "rg-${var.tre_id}-ws-${local.short_workspace_id}" } +data "azurerm_resource_group" "core" { + name = "rg-${var.tre_id}" +} + data "azurerm_virtual_network" "ws" { name = "vnet-${var.tre_id}-ws-${local.short_workspace_id}" resource_group_name = data.azurerm_resource_group.ws.name diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf index d3fff01a8b..540f2763a6 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf @@ -1,7 +1,3 @@ -data "azurerm_resource_group" "core" { - name = "rg-${var.tre_id}" -} - locals { short_service_id = substr(var.tre_resource_id, -4, -1) short_workspace_id = substr(var.workspace_id, -4, -1) diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/main.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/main.tf index 01c5f0b88d..abd02809e1 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/main.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/main.tf @@ -21,6 +21,10 @@ data "azurerm_resource_group" "ws" { name = "rg-${var.tre_id}-ws-${local.short_workspace_id}" } +data "azurerm_resource_group" "core" { + name = "rg-${var.tre_id}" +} + data "azurerm_virtual_network" "ws" { name = "vnet-${var.tre_id}-ws-${local.short_workspace_id}" resource_group_name = data.azurerm_resource_group.ws.name diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/main.tf b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/main.tf index da10b88845..01b0ce69bb 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/main.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/main.tf @@ -21,6 +21,10 @@ data "azurerm_resource_group" "ws" { name = "rg-${var.tre_id}-ws-${local.short_workspace_id}" } +data "azurerm_resource_group" "core" { + name = "rg-${var.tre_id}" +} + data "azurerm_virtual_network" "ws" { name = "vnet-${var.tre_id}-ws-${local.short_workspace_id}" resource_group_name = data.azurerm_resource_group.ws.name From bf5a0bdbc2ab3cc75f6c53286f5b7b228ad1c4d0 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 17:34:54 +0100 Subject: [PATCH 109/142] linting --- .../setup-instructions/configuring-shared-services.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 1ca845c36e..feac429d35 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -2,7 +2,9 @@ ## Deploy/configure Nexus -There is a new Nexus shared service which can be located in the `./templates/shared_services/sonatype-nexus-vm` directory, with the bundle name `tre-shared-service-sonatype-nexus`, which is now hosted using a VM to enable additional configuration required for proxying certain repositories. This has been created as a separate service as the domain name exposed for proxies will be different to the one used by the original Nexus service and thus will break any user resources configured with the old proxy URL. +There is a new Nexus shared service which can be located in the `./templates/shared_services/sonatype-nexus-vm` directory, with the bundle name `tre-shared-service-sonatype-nexus`, which is now hosted using a VM to enable additional configuration required for proxying certain repositories. + +This has been created as a separate service as the domain name exposed for proxies will be different to the one used by the original Nexus service and thus will break any user resources configured with the old proxy URL. The original Nexus service that runs on App Service (located in `./templates/shared_services/sonatype-nexus`) has the bundle name `tre-shared-service-nexus` so can co-exist with the new VM-based shared service to enable smoother upgrading of existing resources. @@ -90,9 +92,11 @@ Once you've created the new VM-based Nexus service by following section `A`, you 1. Identify any existing Guacamole user resources that are using the old proxy URL (`https://nexus-{TRE_ID}.azurewebsites.net/`). These will be any VMs with bundle versions < `0.2.0`. -1. These will need to be either **re-deployed** with the new template versions `0.2.0` or later (which target the new proxy URL format of `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`), or manually have their proxy URLs updated by remoting into the VMs and updating the various configuration files of required package managers with the new URL. For example, pip will need the `index`, `index-url` and `trusted-host` values in the global `pip.conf` file to be modified to use the new URL. +1. These will need to be either **re-deployed** with the new template versions `0.2.0` or later (which target the new proxy URL format of `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`), or manually have their proxy URLs updated by remoting into the VMs and updating the various configuration files of required package managers with the new URL. + + 1. For example, pip will need the `index`, `index-url` and `trusted-host` values in the global `pip.conf` file to be modified to use the new URL. -1. Once you've confirmed there are no dependencies on the old Nexus shared service, you can delete it using the API. +2. Once you've confirmed there are no dependencies on the old Nexus shared service, you can delete it using the API. ## Configure Gitea repositories From e541770f7b638a74dc76c5acc59b987c52b78695 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 12 May 2022 17:48:41 +0100 Subject: [PATCH 110/142] bash linting --- .../shared_services/sonatype-nexus-vm/terraform/deploy.sh | 8 ++++---- .../sonatype-nexus-vm/terraform/destroy.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh b/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh index 007f4acc3c..249a24a15d 100755 --- a/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh +++ b/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh @@ -2,9 +2,9 @@ export TF_LOG="" terraform init -input=false -backend=true -reconfigure \ - -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ - -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ - -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=${TRE_ID}-shared-service-sonatype-nexus" + -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name:?}" \ + -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name:?}" \ + -backend-config="container_name=${TF_VAR_terraform_state_container_name:?}" \ + -backend-config="key=${TRE_ID:?}-shared-service-sonatype-nexus" terraform plan terraform apply -auto-approve diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh b/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh index f93b3830b3..7a19885faf 100755 --- a/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh +++ b/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh @@ -2,9 +2,9 @@ export TF_LOG="" terraform init -input=false -backend=true -reconfigure \ - -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ - -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ - -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=${TRE_ID}-shared-service-sonatype-nexus" + -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name:?}" \ + -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name:?}" \ + -backend-config="container_name=${TF_VAR_terraform_state_container_name:?}" \ + -backend-config="key=${TRE_ID:?}-shared-service-sonatype-nexus" terraform destroy -auto-approve From 010134dcebdd94db80415bdc9337a7fac2baae5e Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 13 May 2022 12:04:54 +0100 Subject: [PATCH 111/142] Require workspace of 0.2.14 or above --- .../setup-instructions/configuring-shared-services.md | 3 +++ templates/shared_services/certs/template_schema.json | 2 +- .../shared_services/sonatype-nexus-vm/template_schema.json | 2 +- templates/workspaces/base/porter.yaml | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index feac429d35..5100632a19 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -15,6 +15,9 @@ If you're deploying a brand new environment you should deploy the new service (r ### A. Deploy & configure new Nexus service (hosted on VM) +!!! caution + Before deploying the new Nexus service, you will need workspaces of version `0.2.14` or above due to a dependency on a DNS zone link for the workspace(s) to connect to the Nexus VM. + Before deploying the Nexus shared service, you need to make sure that it will have access to a certificate to configure serving secure proxies. By default, the Nexus service will serve proxies from `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`, and thus it requires a certificate that validates ownership of this domain to use for SSL. You can use the Certs Shared Service to set one up by following these steps: diff --git a/templates/shared_services/certs/template_schema.json b/templates/shared_services/certs/template_schema.json index b28e76bc5f..266997da81 100644 --- a/templates/shared_services/certs/template_schema.json +++ b/templates/shared_services/certs/template_schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://github.com/microsoft/AzureTRE/templates/shared_services/sonatype-nexus/template_schema.json", + "$id": "https://github.com/microsoft/AzureTRE/templates/shared_services/certs/template_schema.json", "type": "object", "title": "Certs Service", "description": "Provides SSL Certs for a specified internal domain", diff --git a/templates/shared_services/sonatype-nexus-vm/template_schema.json b/templates/shared_services/sonatype-nexus-vm/template_schema.json index dc0685c814..e2d5ca4ec9 100644 --- a/templates/shared_services/sonatype-nexus-vm/template_schema.json +++ b/templates/shared_services/sonatype-nexus-vm/template_schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://github.com/microsoft/AzureTRE/templates/shared_services/sonatype-nexus/template_schema.json", + "$id": "https://github.com/microsoft/AzureTRE/templates/shared_services/sonatype-nexus-vm/template_schema.json", "type": "object", "title": "Nexus Shared Service", "description": "Provides Nexus shared service", diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index 32b0e652f2..a18bae5380 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-base -version: 0.2.13 +version: 0.2.14 description: "A base Azure TRE workspace" registry: azuretre From a042b0b6aee4485f3602d546cadd6a99b9e2fd3f Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 13 May 2022 12:20:11 +0100 Subject: [PATCH 112/142] Moved new version notes to section below config steps --- .../configuring-shared-services.md | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 5100632a19..1f1f31a701 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -2,21 +2,15 @@ ## Deploy/configure Nexus -There is a new Nexus shared service which can be located in the `./templates/shared_services/sonatype-nexus-vm` directory, with the bundle name `tre-shared-service-sonatype-nexus`, which is now hosted using a VM to enable additional configuration required for proxying certain repositories. - -This has been created as a separate service as the domain name exposed for proxies will be different to the one used by the original Nexus service and thus will break any user resources configured with the old proxy URL. - -The original Nexus service that runs on App Service (located in `./templates/shared_services/sonatype-nexus`) has the bundle name `tre-shared-service-nexus` so can co-exist with the new VM-based shared service to enable smoother upgrading of existing resources. - -If you're deploying a brand new environment you should deploy the new service (read section `A`). If you wish to migrate from an existing App Service Nexus service to the new VM service, first deploy the new service (section `A`) then proceed to section `B`. +If you're deploying a brand new environment you should deploy the VM-based service (read section `A`). If you wish to migrate from an existing App Service Nexus service to the VM-based service, first deploy the new service (section `A`) then proceed to section `B`. !!! info - The Makefile commands for deploying shared services temporarily target the old Nexus service so that existing environments won't have a new Nexus service deployed automatically by CICD and introduce breaking changes. The new Nexus service will need to be deployed manually using the steps detailed below. + The Makefile commands for deploying shared services temporarily target the App Service based Nexus service so that existing environments won't have a new Nexus service deployed automatically by CICD and introduce breaking changes. The VM-based Nexus service will need to be deployed manually using the steps detailed below. -### A. Deploy & configure new Nexus service (hosted on VM) +### A. Deploy & configure Nexus service (hosted on VM) !!! caution - Before deploying the new Nexus service, you will need workspaces of version `0.2.14` or above due to a dependency on a DNS zone link for the workspace(s) to connect to the Nexus VM. + Before deploying the VM-based Nexus service, you will need workspaces of version `0.2.14` or above due to a dependency on a DNS zone link for the workspace(s) to connect to the Nexus VM. Before deploying the Nexus shared service, you need to make sure that it will have access to a certificate to configure serving secure proxies. By default, the Nexus service will serve proxies from `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`, and thus it requires a certificate that validates ownership of this domain to use for SSL. @@ -89,7 +83,7 @@ You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE Just bear in mind that if this service is redeployed any changes in the UI won't be persisted. If you wish to add new repositories or alter existing ones, use the JSON files within the `./nexus_repos_config` directory. -### B. Migrate from existing Nexus service (hosted on App Service) +### B. Migrate from an existing Nexus service (hosted on App Service) Once you've created the new VM-based Nexus service by following section `A`, you can migrate from the old App Service Nexus service by following these steps: @@ -101,6 +95,14 @@ Once you've created the new VM-based Nexus service by following section `A`, you 2. Once you've confirmed there are no dependencies on the old Nexus shared service, you can delete it using the API. +### Upgrade notes + +The new Nexus shared service can be located in the `./templates/shared_services/sonatype-nexus-vm` directory, with the bundle name `tre-shared-service-sonatype-nexus`, which is now hosted using a VM to enable additional configuration required for proxying certain repositories. + +This has been created as a separate service as the domain name exposed for proxies will be different to the one used by the original Nexus service and thus will break any user resources configured with the old proxy URL. + +The original Nexus service that runs on App Service (located in `./templates/shared_services/sonatype-nexus`) has the bundle name `tre-shared-service-nexus` so can co-exist with the new VM-based shared service to enable smoother upgrading of existing resources. + ## Configure Gitea repositories Note : This is a Gitea *shared service* which will be accessible from all workspaces intended for mirroring external Git repositories. A Gitea *workspace service* can also be deployed per workspace to enable Gitea to be used within a specific workspace. From 9bd1abf9267fb34afe042609c7b69597393ef434 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 13 May 2022 12:27:20 +0100 Subject: [PATCH 113/142] Removed give new cert name --- .../setup-instructions/configuring-shared-services.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 1f1f31a701..6b85f89ea8 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -41,9 +41,9 @@ You can use the Certs Shared Service to set one up by following these steps: ``` !!! caution - If you have KeyVault Purge Protection enabled and are re-deploying your environment using the same `cert_name`, you may encounter this: `Status=409 Code=\"Conflict\" Message=\"Certificate nexus-ssl is currently in a deleted but recoverable state`. You need to either manually recover the certificate or purge it before redeploying; or alternatively give it a new unique name. + If you have KeyVault Purge Protection enabled and are re-deploying your environment using the same `cert_name`, you may encounter this: `Status=409 Code=\"Conflict\" Message=\"Certificate nexus-ssl is currently in a deleted but recoverable state`. You need to either manually recover the certificate or purge it before redeploying. -4. Once the shared service has been deployed (which you can check by querying the `/api/shared-services/operations` method), copy its `resource_id`, then find the `POST` operation for `/api/shared-services/{shared_service_id}/invoke_action`, click `Try it out` and paste in the resource id into the `shared_service_id` field, and enter `generate` into the `action` field, then click `Execute`. +1. Once the shared service has been deployed (which you can check by querying the `/api/shared-services/operations` method), copy its `resource_id`, then find the `POST` operation for `/api/shared-services/{shared_service_id}/invoke_action`, click `Try it out` and paste in the resource id into the `shared_service_id` field, and enter `generate` into the `action` field, then click `Execute`. This will invoke the certs service to use Letsencrypt to generate a certificate for the specified domain prefix followed by `-{TRE_ID}.{LOCATION}.cloudapp.azure.com`, so in our case, having entered `nexus`, this will be `nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com`, which will be the public domain for our Nexus service. From 6a2f4b022289679dd51243b7f5184672567bda12 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 13 May 2022 14:45:28 +0100 Subject: [PATCH 114/142] RP cert permissions --- templates/core/terraform/resource_processor/vmss_porter/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index dfb134341c..e13951e9d6 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -177,4 +177,5 @@ resource "azurerm_key_vault_access_policy" "resource_processor" { object_id = azurerm_user_assigned_identity.vmss_msi.principal_id secret_permissions = ["Get", "List", "Set", "Delete", "Purge", "Recover"] + certificate_permissions = ["Get", "Recover"] } From 524dd42f3d45dfaa9c252de51c64319adfc132f3 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Fri, 13 May 2022 15:53:26 +0100 Subject: [PATCH 115/142] tf format --- templates/core/terraform/resource_processor/vmss_porter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index e13951e9d6..9fc22147e9 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -176,6 +176,6 @@ resource "azurerm_key_vault_access_policy" "resource_processor" { tenant_id = azurerm_user_assigned_identity.vmss_msi.tenant_id object_id = azurerm_user_assigned_identity.vmss_msi.principal_id - secret_permissions = ["Get", "List", "Set", "Delete", "Purge", "Recover"] + secret_permissions = ["Get", "List", "Set", "Delete", "Purge", "Recover"] certificate_permissions = ["Get", "Recover"] } From 81eba218bd839f0cda055618be767bceb150c95b Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 23 May 2022 13:30:17 +0000 Subject: [PATCH 116/142] Added required params for certs and nexus tempalte schema --- templates/shared_services/certs/template_schema.json | 5 ++++- .../shared_services/sonatype-nexus-vm/template_schema.json | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/templates/shared_services/certs/template_schema.json b/templates/shared_services/certs/template_schema.json index 266997da81..6a10beb70d 100644 --- a/templates/shared_services/certs/template_schema.json +++ b/templates/shared_services/certs/template_schema.json @@ -4,7 +4,10 @@ "type": "object", "title": "Certs Service", "description": "Provides SSL Certs for a specified internal domain", - "required": [], + "required": [ + "domain_prefix", + "cert_name" + ], "properties": { "domain_prefix": { "$id": "#/properties/domain_prefix", diff --git a/templates/shared_services/sonatype-nexus-vm/template_schema.json b/templates/shared_services/sonatype-nexus-vm/template_schema.json index e2d5ca4ec9..baf7e777a7 100644 --- a/templates/shared_services/sonatype-nexus-vm/template_schema.json +++ b/templates/shared_services/sonatype-nexus-vm/template_schema.json @@ -4,13 +4,14 @@ "type": "object", "title": "Nexus Shared Service", "description": "Provides Nexus shared service", - "required": [], + "required": [ + "ssl_cert_name" + ], "properties": { "ssl_cert_name": { "type": "string", "title": "SSL certificate name", - "description": "The name of the certificate to use (located in the core KeyVault) for configuring Nexus SSL", - "default": "nexus-ssl" + "description": "The name of the certificate to use (located in the core KeyVault) for configuring Nexus SSL" } } } From 30892556c38b3dfc83d12e3942a8797d66e241eb Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 23 May 2022 13:36:55 +0000 Subject: [PATCH 117/142] Added cert import permissions --- templates/core/terraform/resource_processor/vmss_porter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index 9fc22147e9..fce8c7d0ba 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -177,5 +177,5 @@ resource "azurerm_key_vault_access_policy" "resource_processor" { object_id = azurerm_user_assigned_identity.vmss_msi.principal_id secret_permissions = ["Get", "List", "Set", "Delete", "Purge", "Recover"] - certificate_permissions = ["Get", "Recover"] + certificate_permissions = ["Get", "Recover", "Import"] } From 8e310367af439d3c5049f40b5a9e816621306b0c Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 23 May 2022 13:38:32 +0000 Subject: [PATCH 118/142] Added certs delete permission --- templates/core/terraform/resource_processor/vmss_porter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index fce8c7d0ba..a30a54f9fe 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -177,5 +177,5 @@ resource "azurerm_key_vault_access_policy" "resource_processor" { object_id = azurerm_user_assigned_identity.vmss_msi.principal_id secret_permissions = ["Get", "List", "Set", "Delete", "Purge", "Recover"] - certificate_permissions = ["Get", "Recover", "Import"] + certificate_permissions = ["Get", "Recover", "Import", "Delete"] } From b5650f53187bbedaa818a45520bc7951f9f76946 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 23 May 2022 16:07:18 +0000 Subject: [PATCH 119/142] App gateway az login --- .../certs/terraform/appgateway.tf | 26 +++++++++++++++++++ .../shared_services/certs/terraform/data.tf | 2 ++ .../certs/terraform/variables.tf | 8 ++++++ 3 files changed, 36 insertions(+) diff --git a/templates/shared_services/certs/terraform/appgateway.tf b/templates/shared_services/certs/terraform/appgateway.tf index 80afa5a7ec..2354ef7c7d 100644 --- a/templates/shared_services/certs/terraform/appgateway.tf +++ b/templates/shared_services/certs/terraform/appgateway.tf @@ -1,3 +1,25 @@ +resource "null_resource" "az_login_sp" { + count = var.arm_use_msi == true ? 0 : 1 + provisioner "local-exec" { + command = "az login --service-principal --username ${var.arm_client_id} --password ${var.arm_client_secret} --tenant ${var.arm_tenant_id}" + } + + triggers = { + timestamp = timestamp() + } +} + +resource "null_resource" "az_login_msi" { + count = var.arm_use_msi == true ? 1 : 0 + provisioner "local-exec" { + command = "az login --identity -u '${data.azurerm_client_config.current.client_id}'" + } + + triggers = { + timestamp = timestamp() + } +} + resource "azurerm_public_ip" "appgwpip" { name = "pip-cert-${var.domain_prefix}-${var.tre_id}" resource_group_name = data.azurerm_resource_group.rg.name @@ -163,6 +185,10 @@ resource "azurerm_application_gateway" "agw" { # Stop app gateway once provisioned to save cost until the generate custom action is invoked (which will start/stop as required) provisioner "local-exec" { command = "az network application-gateway stop -g ${data.azurerm_resource_group.rg.name} -n agw-certs-${var.tre_id}" + depends_on = [ + null_resource.az_login_sp, + null_resource.az_login_msi + ] } } diff --git a/templates/shared_services/certs/terraform/data.tf b/templates/shared_services/certs/terraform/data.tf index 47db962c89..9de429226f 100644 --- a/templates/shared_services/certs/terraform/data.tf +++ b/templates/shared_services/certs/terraform/data.tf @@ -1,3 +1,5 @@ +data "azurerm_client_config" "current" {} + data "azurerm_resource_group" "rg" { name = "rg-${var.tre_id}" } diff --git a/templates/shared_services/certs/terraform/variables.tf b/templates/shared_services/certs/terraform/variables.tf index cc3391ad14..9a493e14a3 100644 --- a/templates/shared_services/certs/terraform/variables.tf +++ b/templates/shared_services/certs/terraform/variables.tf @@ -2,6 +2,14 @@ variable "tre_id" { type = string } +variable "arm_use_msi" { + type = bool +} + +variable "arm_tenant_id" {} +variable "arm_client_id" {} +variable "arm_client_secret" {} + variable "domain_prefix" { type = string } From 15d3fd38c4cdfb2d89322b33ead17647d83062c1 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 23 May 2022 16:13:33 +0000 Subject: [PATCH 120/142] Version bumps --- templates/shared_services/certs/porter.yaml | 2 +- templates/shared_services/sonatype-nexus-vm/porter.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index 3f589488f8..0fcc9fa42d 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-certs -version: 0.0.10 +version: 0.0.11 description: "An Azure TRE shared service to generate certificates for a specified internal domain using Letsencrypt" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/shared_services/sonatype-nexus-vm/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml index cb15837b36..f92ced6d5e 100644 --- a/templates/shared_services/sonatype-nexus-vm/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-sonatype-nexus -version: 1.0.1 +version: 1.0.2 description: "A Sonatype Nexus shared service" registry: azuretre credentials: From 65f4138bfcbc70f6f8095d0ba35f7219b4f2daaf Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 23 May 2022 19:53:05 +0000 Subject: [PATCH 121/142] tf fmt --- templates/shared_services/certs/terraform/appgateway.tf | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/templates/shared_services/certs/terraform/appgateway.tf b/templates/shared_services/certs/terraform/appgateway.tf index 2354ef7c7d..d3f2ad5291 100644 --- a/templates/shared_services/certs/terraform/appgateway.tf +++ b/templates/shared_services/certs/terraform/appgateway.tf @@ -179,16 +179,14 @@ resource "azurerm_application_gateway" "agw" { } depends_on = [ - azurerm_key_vault_access_policy.app_gw_managed_identity + azurerm_key_vault_access_policy.app_gw_managed_identity, + null_resource.az_login_sp, + null_resource.az_login_msi ] # Stop app gateway once provisioned to save cost until the generate custom action is invoked (which will start/stop as required) provisioner "local-exec" { command = "az network application-gateway stop -g ${data.azurerm_resource_group.rg.name} -n agw-certs-${var.tre_id}" - depends_on = [ - null_resource.az_login_sp, - null_resource.az_login_msi - ] } } From 60a4976c7dd239026abaaeded16470d37c0f5428 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Mon, 23 May 2022 22:19:56 +0000 Subject: [PATCH 122/142] Added missing az cred params to certs --- templates/shared_services/certs/porter.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index 0fcc9fa42d..8e5664472f 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -52,6 +52,10 @@ install: input: false vars: tre_id: "{{ bundle.parameters.tre_id }}" + arm_tenant_id: "{{ bundle.credentials.azure_tenant_id }}" + arm_client_id: "{{ bundle.credentials.azure_client_id }}" + arm_client_secret: "{{ bundle.credentials.azure_client_secret }}" + arm_use_msi: "{{ bundle.parameters.arm_use_msi }}" domain_prefix: "{{ bundle.parameters.domain_prefix }}" cert_name: "{{ bundle.parameters.cert_name }}" backendConfig: @@ -77,6 +81,10 @@ uninstall: input: false vars: tre_id: "{{ bundle.parameters.tre_id }}" + arm_tenant_id: "{{ bundle.credentials.azure_tenant_id }}" + arm_client_id: "{{ bundle.credentials.azure_client_id }}" + arm_client_secret: "{{ bundle.credentials.azure_client_secret }}" + arm_use_msi: "{{ bundle.parameters.arm_use_msi }}" domain_prefix: "{{ bundle.parameters.domain_prefix }}" cert_name: "{{ bundle.parameters.cert_name }}" backendConfig: From d48c4cf452d0a5ca51010f1750722f630641d3d3 Mon Sep 17 00:00:00 2001 From: marrobi Date: Tue, 24 May 2022 07:44:49 +0000 Subject: [PATCH 123/142] Add purge permission --- templates/core/terraform/resource_processor/vmss_porter/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/core/terraform/resource_processor/vmss_porter/main.tf b/templates/core/terraform/resource_processor/vmss_porter/main.tf index a30a54f9fe..9f9a328303 100644 --- a/templates/core/terraform/resource_processor/vmss_porter/main.tf +++ b/templates/core/terraform/resource_processor/vmss_porter/main.tf @@ -177,5 +177,5 @@ resource "azurerm_key_vault_access_policy" "resource_processor" { object_id = azurerm_user_assigned_identity.vmss_msi.principal_id secret_permissions = ["Get", "List", "Set", "Delete", "Purge", "Recover"] - certificate_permissions = ["Get", "Recover", "Import", "Delete"] + certificate_permissions = ["Get", "Recover", "Import", "Delete", "Purge"] } From 68e5c096c225824718917be99a3b1cced52d6f76 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 24 May 2022 13:24:32 +0000 Subject: [PATCH 124/142] Bump tf versions to 3.4.0 & set purge to false --- templates/shared_services/certs/terraform/main.tf | 11 +++++++++-- .../sonatype-nexus-vm/terraform/main.tf | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/templates/shared_services/certs/terraform/main.tf b/templates/shared_services/certs/terraform/main.tf index 715a8db036..2ead62214d 100644 --- a/templates/shared_services/certs/terraform/main.tf +++ b/templates/shared_services/certs/terraform/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "=2.97.0" + version = "=3.4.0" } } @@ -11,5 +11,12 @@ terraform { } provider "azurerm" { - features {} + features { + key_vault { + # Don't purge secrets on destroy (this would fail due to purge protection being enabled on keyvault) + purge_soft_deleted_secrets_on_destroy = false + # When recreating a shared service, recover any previously soft deleted secrets + recover_soft_deleted_secrets = true + } + } } diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/main.tf b/templates/shared_services/sonatype-nexus-vm/terraform/main.tf index 715a8db036..2ead62214d 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/main.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "=2.97.0" + version = "=3.4.0" } } @@ -11,5 +11,12 @@ terraform { } provider "azurerm" { - features {} + features { + key_vault { + # Don't purge secrets on destroy (this would fail due to purge protection being enabled on keyvault) + purge_soft_deleted_secrets_on_destroy = false + # When recreating a shared service, recover any previously soft deleted secrets + recover_soft_deleted_secrets = true + } + } } From 3b3475555fd89072ec835659f2754f2a77222387 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 24 May 2022 13:59:59 +0000 Subject: [PATCH 125/142] Removed unsupported property from new provider --- templates/shared_services/certs/terraform/staticweb.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/shared_services/certs/terraform/staticweb.tf b/templates/shared_services/certs/terraform/staticweb.tf index d6a315cd89..5b8445c46d 100644 --- a/templates/shared_services/certs/terraform/staticweb.tf +++ b/templates/shared_services/certs/terraform/staticweb.tf @@ -7,7 +7,6 @@ resource "azurerm_storage_account" "staticweb" { account_tier = "Standard" account_replication_type = "LRS" enable_https_traffic_only = true - allow_blob_public_access = true tags = { tre_id = var.tre_id From f5fc4123c3803d961f9c548684d13ebc86a8ccc0 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 24 May 2022 14:25:48 +0000 Subject: [PATCH 126/142] Moved nexus private zone to core --- templates/core/terraform/network/dns_zones.tf | 7 +++++++ .../sonatype-nexus-vm/terraform/data.tf | 5 +++++ .../shared_services/sonatype-nexus-vm/terraform/vm.tf | 11 ++--------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/templates/core/terraform/network/dns_zones.tf b/templates/core/terraform/network/dns_zones.tf index 5dc8c55cf5..7c4f479732 100644 --- a/templates/core/terraform/network/dns_zones.tf +++ b/templates/core/terraform/network/dns_zones.tf @@ -225,3 +225,10 @@ resource "azurerm_private_dns_zone" "postgres" { lifecycle { ignore_changes = [tags] } } + +resource "azurerm_private_dns_zone" "nexus" { + name = "nexus-${var.tre_id}.${data.azurerm_resource_group.rg.location}.cloudapp.azure.com" + resource_group_name = local.core_resource_group_name + + lifecycle { ignore_changes = [tags] } +} diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/data.tf b/templates/shared_services/sonatype-nexus-vm/terraform/data.tf index fc5031c4ce..3e292fffbd 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/data.tf @@ -42,3 +42,8 @@ data "azurerm_storage_account" "nexus" { data "azurerm_resource_group" "rg" { name = local.core_resource_group_name } + +data "azurerm_private_dns_zone" "nexus" { + name = "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" + resource_group_name = local.core_resource_group_name +} diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf index fb88d44817..5d2bd8e2fa 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf @@ -10,23 +10,16 @@ resource "azurerm_network_interface" "nexus" { } } -resource "azurerm_private_dns_zone" "nexus" { - name = "nexus-${var.tre_id}.${data.azurerm_resource_group.rg.location}.cloudapp.azure.com" - resource_group_name = local.core_resource_group_name - - lifecycle { ignore_changes = [tags] } -} - resource "azurerm_private_dns_zone_virtual_network_link" "nexus_core_vnet" { name = "nexuslink-core" resource_group_name = local.core_resource_group_name - private_dns_zone_name = azurerm_private_dns_zone.nexus.name + private_dns_zone_name = data.azurerm_private_dns_zone.nexus.name virtual_network_id = data.azurerm_virtual_network.core.id } resource "azurerm_private_dns_a_record" "nexus_vm" { name = "@" - zone_name = azurerm_private_dns_zone.nexus.name + zone_name = data.azurerm_private_dns_zone.nexus.name resource_group_name = local.core_resource_group_name ttl = 300 records = [azurerm_linux_virtual_machine.nexus.private_ip_address] From d951785e6dbef2c16288b1c7ee106ed9e9550aa8 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 24 May 2022 14:30:08 +0000 Subject: [PATCH 127/142] Amended location var --- templates/core/terraform/network/dns_zones.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/core/terraform/network/dns_zones.tf b/templates/core/terraform/network/dns_zones.tf index 7c4f479732..f70583dec7 100644 --- a/templates/core/terraform/network/dns_zones.tf +++ b/templates/core/terraform/network/dns_zones.tf @@ -227,7 +227,7 @@ resource "azurerm_private_dns_zone" "postgres" { } resource "azurerm_private_dns_zone" "nexus" { - name = "nexus-${var.tre_id}.${data.azurerm_resource_group.rg.location}.cloudapp.azure.com" + name = "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" resource_group_name = local.core_resource_group_name lifecycle { ignore_changes = [tags] } From 9c3df192aaf980ed6ab80cf6ef39b05f05bacd54 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 24 May 2022 14:31:32 +0000 Subject: [PATCH 128/142] Amended zone location --- templates/core/terraform/network/dns_zones.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/core/terraform/network/dns_zones.tf b/templates/core/terraform/network/dns_zones.tf index f70583dec7..e42a84b24c 100644 --- a/templates/core/terraform/network/dns_zones.tf +++ b/templates/core/terraform/network/dns_zones.tf @@ -228,7 +228,7 @@ resource "azurerm_private_dns_zone" "postgres" { resource "azurerm_private_dns_zone" "nexus" { name = "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" - resource_group_name = local.core_resource_group_name + resource_group_name = var.resource_group_name lifecycle { ignore_changes = [tags] } } From 621d471c59430f46582e31c9a679a36c189101c4 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Tue, 24 May 2022 15:27:06 +0000 Subject: [PATCH 129/142] Added upgrade flag for tf --- templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh b/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh index 249a24a15d..2a371d27e3 100755 --- a/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh +++ b/templates/shared_services/sonatype-nexus-vm/terraform/deploy.sh @@ -1,7 +1,7 @@ #!/bin/bash export TF_LOG="" -terraform init -input=false -backend=true -reconfigure \ +terraform init -input=false -backend=true -reconfigure -upgrade \ -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name:?}" \ -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name:?}" \ -backend-config="container_name=${TF_VAR_terraform_state_container_name:?}" \ From 13bc8a1b0086de83351ad96fde5b99d350660d70 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 25 May 2022 11:22:15 +0000 Subject: [PATCH 130/142] Remove tf lock --- .../configuring-shared-services.md | 2 +- .../terraform/.terraform.lock.hcl | 57 ------------------- .../sonatype-nexus-vm/terraform/data.tf | 2 +- .../sonatype-nexus-vm/terraform/destroy.sh | 2 +- 4 files changed, 3 insertions(+), 60 deletions(-) delete mode 100644 templates/shared_services/sonatype-nexus-vm/terraform/.terraform.lock.hcl diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 6b85f89ea8..71ca38095b 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -5,7 +5,7 @@ If you're deploying a brand new environment you should deploy the VM-based service (read section `A`). If you wish to migrate from an existing App Service Nexus service to the VM-based service, first deploy the new service (section `A`) then proceed to section `B`. !!! info - The Makefile commands for deploying shared services temporarily target the App Service based Nexus service so that existing environments won't have a new Nexus service deployed automatically by CICD and introduce breaking changes. The VM-based Nexus service will need to be deployed manually using the steps detailed below. + The Makefile commands for deploying shared services temporarily target the App Service Nexus service so that existing environments won't have a new Nexus service deployed automatically by CICD and introduce breaking changes. The VM-based Nexus service will need to be deployed manually using the steps below and is required when deploying new Guacamole user resources of version `0.2.0` or higher. ### A. Deploy & configure Nexus service (hosted on VM) diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/.terraform.lock.hcl b/templates/shared_services/sonatype-nexus-vm/terraform/.terraform.lock.hcl deleted file mode 100644 index 0051ace45a..0000000000 --- a/templates/shared_services/sonatype-nexus-vm/terraform/.terraform.lock.hcl +++ /dev/null @@ -1,57 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "2.97.0" - constraints = "2.97.0" - hashes = [ - "h1:XxT+XM/leTXa21aTnJjPBfNBQ8cLE4gYDg01WEZsV1U=", - "zh:0aac80e6d2b8ddf33d558ac893d52688e8abf8a0b995cfc3c35eb84afbf432a3", - "zh:11191068cb732208ebc8662651782f63db329a25f7ea1cd50cd91622a2c247b7", - "zh:36c8334194e7d605682053c7c70fbb2a650d9b0a7bcc44d5cdda4f205818438a", - "zh:3a5e01276added995e875b42ecc6b36ff73d267f0c096c87195bd2b1fff4f5b2", - "zh:557e38371657e6ed8aae9192d01480c4cca7c0f7ade6022f1aec247a6384922b", - "zh:67b913c280c5858549477a4b05e77078b1a5234de77c7bddd4ee1e8e237d5665", - "zh:7aeca864ce45b295db734cd968f7596ff12cd7c522ee89d53f432dae7c2b5d18", - "zh:b6127d7a796eaf9756dd212667eb48f79c0e78729589ec8ccf68e0b36ebb4e54", - "zh:bed448238740f897d1b399e5123b3a9eba256b981846f9ee92b71493446ca684", - "zh:c351a1bba34c3bd06fff75e4c15e4db0456268479463c2471598068ea1c5c884", - "zh:d073c24d0a4756e79b39f41f552d526800f9fb0ad0a74f742ac8de61b6416a3a", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.1.2" - hashes = [ - "h1:5A5VsY5wNmOZlupUcLnIoziMPn8htSZBXbP3lI7lBEM=", - "zh:0daceba867b330d3f8e2c5dc895c4291845a78f31955ce1b91ab2c4d1cd1c10b", - "zh:104050099efd30a630741f788f9576b19998e7a09347decbec3da0b21d64ba2d", - "zh:173f4ef3fdf0c7e2564a3db0fac560e9f5afdf6afd0b75d6646af6576b122b16", - "zh:41d50f975e535f968b3f37170fb07937c15b76d85ba947d0ce5e5ff9530eda65", - "zh:51a5038867e5e60757ed7f513dd6a973068241190d158a81d1b69296efb9cb8d", - "zh:6432a568e97a5a36cc8aebca5a7e9c879a55d3bc71d0da1ab849ad905f41c0be", - "zh:6bac6501394b87138a5e17c9f3a41e46ff7833ad0ba2a96197bb7787e95b641c", - "zh:6c0a7f5faacda644b022e7718e53f5868187435be6d000786d1ca05aa6683a25", - "zh:74c89de3fa6ef3027efe08f8473c2baeb41b4c6cee250ba7aeb5b64e8c79800d", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:b29eabbf0a5298f0e95a1df214c7cfe06ea9bcf362c63b3ad2f72d85da7d4685", - "zh:e891458c7a61e5b964e09616f1a4f87d0471feae1ec04cc51776e7dec1a3abce", - ] -} - -provider "registry.terraform.io/hashicorp/template" { - version = "2.2.0" - hashes = [ - "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", - "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", - "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", - "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", - "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", - "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", - "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", - "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", - "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", - "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", - "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", - ] -} diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/data.tf b/templates/shared_services/sonatype-nexus-vm/terraform/data.tf index 3e292fffbd..275a017f8e 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/data.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/data.tf @@ -44,6 +44,6 @@ data "azurerm_resource_group" "rg" { } data "azurerm_private_dns_zone" "nexus" { - name = "nexus-${var.tre_id}.${var.location}.cloudapp.azure.com" + name = "nexus-${var.tre_id}.${data.azurerm_resource_group.rg.location}.cloudapp.azure.com" resource_group_name = local.core_resource_group_name } diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh b/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh index 7a19885faf..853a7142f3 100755 --- a/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh +++ b/templates/shared_services/sonatype-nexus-vm/terraform/destroy.sh @@ -1,7 +1,7 @@ #!/bin/bash export TF_LOG="" -terraform init -input=false -backend=true -reconfigure \ +terraform init -input=false -backend=true -reconfigure -upgrade \ -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name:?}" \ -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name:?}" \ -backend-config="container_name=${TF_VAR_terraform_state_container_name:?}" \ From f6705b25cd091c47f23484e583f088c480b3b73e Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 25 May 2022 12:03:13 +0000 Subject: [PATCH 131/142] Added new tf key --- templates/shared_services/sonatype-nexus-vm/porter.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/shared_services/sonatype-nexus-vm/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml index f92ced6d5e..cce4a4981f 100644 --- a/templates/shared_services/sonatype-nexus-vm/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-sonatype-nexus -version: 1.0.2 +version: 1.0.3 description: "A Sonatype Nexus shared service" registry: azuretre credentials: @@ -52,7 +52,7 @@ install: storage_account_name: "{{ bundle.parameters.tfstate_storage_account_name }}" container_name: "{{ bundle.parameters.tfstate_container_name }}" - key: "{{ bundle.parameters.tre_id }}-shared-service-sonatype-nexus" + key: "{{ bundle.parameters.tre_id }}-shared-service-sonatype-nexus-vm" upgrade: - exec: description: "Upgrade shared service" From d933c9122c6a7b5f977dec0be4dfa90e3df89da0 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 25 May 2022 12:05:14 +0000 Subject: [PATCH 132/142] Added key into uninstall --- templates/shared_services/sonatype-nexus-vm/porter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus-vm/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml index cce4a4981f..b9affa34c8 100644 --- a/templates/shared_services/sonatype-nexus-vm/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -72,4 +72,4 @@ uninstall: storage_account_name: "{{ bundle.parameters.tfstate_storage_account_name }}" container_name: "{{ bundle.parameters.tfstate_container_name }}" - key: "{{ bundle.parameters.tre_id }}-shared-service-sonatype-nexus" + key: "{{ bundle.parameters.tre_id }}-shared-service-sonatype-nexus-vm" From 57427e1c90bd44c381e565c879e05bcbd14bdbf6 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 25 May 2022 15:17:04 +0000 Subject: [PATCH 133/142] Resolve firewall rule conflicts --- templates/shared_services/sonatype-nexus-vm/porter.yaml | 2 +- .../shared_services/sonatype-nexus-vm/terraform/firewall.tf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/shared_services/sonatype-nexus-vm/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml index b9affa34c8..f0c5941c7a 100644 --- a/templates/shared_services/sonatype-nexus-vm/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-sonatype-nexus -version: 1.0.3 +version: 1.0.4 description: "A Sonatype Nexus shared service" registry: azuretre credentials: diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/firewall.tf b/templates/shared_services/sonatype-nexus-vm/terraform/firewall.tf index 092f7c3a43..5b5d26f2d2 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/firewall.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/firewall.tf @@ -1,8 +1,8 @@ -resource "azurerm_firewall_application_rule_collection" "shared_subnet_nexus" { - name = "shared_subnet_nexus" +resource "azurerm_firewall_application_rule_collection" "shared_subnet_sonatype_nexus" { + name = "shared_subnet_sonatype_nexus" azure_firewall_name = data.azurerm_firewall.fw.name resource_group_name = data.azurerm_firewall.fw.resource_group_name - priority = 104 + priority = 105 action = "Allow" rule { From f87d03997f0be3d1242ce4c6857562a98c064e60 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 25 May 2022 16:15:57 +0000 Subject: [PATCH 134/142] Var reference fix --- templates/shared_services/sonatype-nexus-vm/terraform/vm.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf index 5d2bd8e2fa..945b880889 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf @@ -116,7 +116,7 @@ resource "azurerm_linux_virtual_machine" "nexus" { depends_on = [ azurerm_key_vault_access_policy.nexus_msi, - azurerm_firewall_application_rule_collection.shared_subnet_nexus + azurerm_firewall_application_rule_collection.shared_subnet_sonatype_nexus ] connection { From a8a7aec990edf8842adca5ce54e21ae99fab5e3f Mon Sep 17 00:00:00 2001 From: James Griffin Date: Wed, 25 May 2022 17:50:07 +0000 Subject: [PATCH 135/142] Fix for potential @ symbol in nexus admin password causing curl bug --- templates/shared_services/sonatype-nexus-vm/terraform/vm.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf index 945b880889..301b70ac40 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf @@ -48,7 +48,7 @@ resource "random_password" "nexus_admin_password" { min_numeric = 1 special = true min_special = 1 - override_special = "_%@" + override_special = "_%" } resource "azurerm_key_vault_secret" "nexus_vm_password" { From 9ffb315762c8cb384c7936ac3decc46dfe17aa4b Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 26 May 2022 11:51:20 +0000 Subject: [PATCH 136/142] Added nexus_version variable to user resources for back compat --- .../shared_services/sonatype-nexus-vm/porter.yaml | 2 +- .../guacamole-azure-linuxvm/porter.yaml | 9 ++++++++- .../guacamole-azure-linuxvm/template_schema.json | 11 +++++++++++ .../guacamole-azure-linuxvm/terraform/linuxvm.tf | 6 +++--- .../guacamole-azure-linuxvm/terraform/locals.tf | 6 +++++- .../guacamole-azure-linuxvm/terraform/variables.tf | 1 + .../guacamole-azure-windowsvm/porter.yaml | 10 +++++++++- .../guacamole-azure-windowsvm/template_schema.json | 11 +++++++++++ .../guacamole-azure-windowsvm/terraform/locals.tf | 6 +++++- .../guacamole-azure-windowsvm/terraform/variables.tf | 1 + .../guacamole-azure-windowsvm/terraform/windowsvm.tf | 2 +- .../user_resources/guacamole-dev-vm/porter.yaml | 9 ++++++++- .../guacamole-dev-vm/template_schema.json | 11 +++++++++++ .../guacamole-dev-vm/terraform/linuxvm.tf | 6 +++--- .../guacamole-dev-vm/terraform/locals.tf | 6 +++++- .../guacamole-dev-vm/terraform/variables.tf | 1 + templates/workspaces/base/porter.yaml | 2 +- 17 files changed, 85 insertions(+), 15 deletions(-) diff --git a/templates/shared_services/sonatype-nexus-vm/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml index f0c5941c7a..5cfc16b8eb 100644 --- a/templates/shared_services/sonatype-nexus-vm/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-shared-service-sonatype-nexus -version: 1.0.4 +version: 2.0.0 description: "A Sonatype Nexus shared service" registry: azuretre credentials: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index 46a3a67fdb..5520924be6 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-linuxvm -version: 0.3.0 +version: 0.3.2 description: "An Azure TRE User Resource Template for Guacamole (Linux)" registry: azuretre dockerfile: Dockerfile.tmpl @@ -56,6 +56,10 @@ parameters: - name: shared_storage_name type: string default: "vm-shared-storage" + - name: nexus_version + type: string + default: "V1" + description: "Which Nexus proxy service to use, i.e. V1 for the App Service-based Nexus or V2 for the VM-based service" outputs: - name: ip @@ -98,6 +102,7 @@ install: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -126,6 +131,7 @@ upgrade: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -154,6 +160,7 @@ uninstall: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json index 7e5027675f..27db5c3d0f 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json @@ -29,6 +29,17 @@ "16 CPU | 64GB RAM" ], "updateable": true + }, + "nexus_version": { + "$id": "#/properties/nexus_version", + "type": "string", + "title": "Nexus", + "description": "Which Nexus proxy service to use, i.e. V1 for the App Service-based Nexus or V2 for the VM-based service", + "enum": [ + "V1", + "V2" + ], + "default": "V1" } } } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf index 7ae293288b..dc929a2ddc 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf @@ -97,7 +97,7 @@ data "template_file" "vm_config" { storage_account_key = data.azurerm_storage_account.stg.primary_access_key http_endpoint = data.azurerm_storage_account.stg.primary_file_endpoint fileshare_name = data.azurerm_storage_share.shared_storage.name - nexus_proxy_url = local.nexus_proxy_url + nexus_proxy_url = local.nexus_proxy_url[var.nexus_version] conda_config = local.image_ref[var.image].conda_config ? 1 : 0 } } @@ -105,14 +105,14 @@ data "template_file" "vm_config" { data "template_file" "pypi_sources_config" { template = file("${path.module}/pypi_sources_config.sh") vars = { - nexus_proxy_url = local.nexus_proxy_url + nexus_proxy_url = local.nexus_proxy_url[var.nexus_version] } } data "template_file" "apt_sources_config" { template = file("${path.module}/apt_sources_config.yml") vars = { - nexus_proxy_url = local.nexus_proxy_url + nexus_proxy_url = local.nexus_proxy_url[var.nexus_version] } } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf index a2d35fef10..1f75144c03 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf @@ -9,7 +9,11 @@ locals { vm_name = "linuxvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") storage_name = lower(replace("stg${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", "")) - nexus_proxy_url = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" + + nexus_proxy_url = { + "V1" = "https://nexus-${var.tre_id}.azurewebsites.net", + "V2" = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" + } vm_size = { "2 CPU | 8GB RAM" = { value = "Standard_D2s_v5" }, "4 CPU | 16GB RAM" = { value = "Standard_D4s_v5" }, diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf index 11b9a24565..3f3fd36c4d 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/variables.tf @@ -12,3 +12,4 @@ variable "image" {} variable "vm_size" {} variable "shared_storage_access" {} variable "shared_storage_name" {} +variable "nexus_version" {} diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index 7c6911c456..c904a2fefd 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-guacamole-windowsvm -version: 0.3.0 +version: 0.3.2 description: "An Azure TRE User Resource Template for Guacamole (Windows 10)" registry: azuretre dockerfile: Dockerfile.tmpl @@ -56,6 +56,11 @@ parameters: - name: shared_storage_name type: string default: "vm-shared-storage" + - name: nexus_version + type: string + default: "V1" + description: "Which Nexus proxy service to use, i.e. V1 for the App Service-based Nexus or V2 for the VM-based service" + outputs: - name: ip @@ -98,6 +103,7 @@ install: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -126,6 +132,7 @@ upgrade: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -154,6 +161,7 @@ uninstall: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/template_schema.json b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/template_schema.json index f4eb91c241..2e7f2a2433 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/template_schema.json +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/template_schema.json @@ -29,6 +29,17 @@ "16 CPU | 64GB RAM" ], "updateable": true + }, + "nexus_version": { + "$id": "#/properties/nexus_version", + "type": "string", + "title": "Nexus", + "description": "Which Nexus proxy service to use, i.e. V1 for the App Service-based Nexus or V2 for the VM-based service", + "enum": [ + "V1", + "V2" + ], + "default": "V1" } } } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf index 540f2763a6..43e0ffec35 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf @@ -9,7 +9,11 @@ locals { vm_name = "windowsvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") storage_name = lower(replace("stg${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", "")) - nexus_proxy_url = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" + + nexus_proxy_url = { + "V1" = "https://nexus-${var.tre_id}.azurewebsites.net", + "V2" = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" + } vm_size = { "2 CPU | 8GB RAM" = { value = "Standard_D2s_v5" }, "4 CPU | 16GB RAM" = { value = "Standard_D4s_v5" }, diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf index 11b9a24565..3f3fd36c4d 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/variables.tf @@ -12,3 +12,4 @@ variable "image" {} variable "vm_size" {} variable "shared_storage_access" {} variable "shared_storage_name" {} +variable "nexus_version" {} diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf index d61bd09708..10a036d5a1 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf @@ -90,7 +90,7 @@ resource "azurerm_key_vault_secret" "windowsvm_password" { data "template_file" "vm_config" { template = file("${path.module}/vm_config.ps1") vars = { - nexus_proxy_url = local.nexus_proxy_url + nexus_proxy_url = local.nexus_proxy_url[var.nexus_version] SharedStorageAccess = tobool(var.shared_storage_access) ? 1 : 0 StorageAccountName = data.azurerm_storage_account.stg.name StorageAccountKey = data.azurerm_storage_account.stg.primary_access_key diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/porter.yaml index 1f455a82d5..116073f4df 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-service-dev-vm -version: 0.3.1 +version: 0.3.2 description: "An Azure TRE User Resource Template for a Dev VM" registry: azuretre dockerfile: Dockerfile.tmpl @@ -56,6 +56,10 @@ parameters: - name: shared_storage_name type: string default: "vm-shared-storage" + - name: nexus_version + type: string + default: "V1" + description: "Which Nexus proxy service to use, i.e. V1 for the App Service-based Nexus or V2 for the VM-based service" outputs: - name: ip @@ -98,6 +102,7 @@ install: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -126,6 +131,7 @@ upgrade: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" @@ -154,6 +160,7 @@ uninstall: vm_size: "{{ bundle.parameters.vm_size }}" shared_storage_access: "{{ bundle.parameters.shared_storage_access }}" shared_storage_name: "{{ bundle.parameters.shared_storage_name }}" + nexus_version: "{{ bundle.parameters.nexus_version }}" backendConfig: resource_group_name: "{{ bundle.parameters.tfstate_resource_group_name }}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/template_schema.json b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/template_schema.json index e2eedfe1db..badd6cb24d 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/template_schema.json +++ b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/template_schema.json @@ -29,6 +29,17 @@ "16 CPU | 64GB RAM" ], "updateable": true + }, + "nexus_version": { + "$id": "#/properties/nexus_version", + "type": "string", + "title": "Nexus", + "description": "Which Nexus proxy service to use, i.e. V1 for the App Service-based Nexus or V2 for the VM-based service", + "enum": [ + "V1", + "V2" + ], + "default": "V1" } }, "pipeline": { diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/linuxvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/linuxvm.tf index 7ae293288b..dc929a2ddc 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/linuxvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/linuxvm.tf @@ -97,7 +97,7 @@ data "template_file" "vm_config" { storage_account_key = data.azurerm_storage_account.stg.primary_access_key http_endpoint = data.azurerm_storage_account.stg.primary_file_endpoint fileshare_name = data.azurerm_storage_share.shared_storage.name - nexus_proxy_url = local.nexus_proxy_url + nexus_proxy_url = local.nexus_proxy_url[var.nexus_version] conda_config = local.image_ref[var.image].conda_config ? 1 : 0 } } @@ -105,14 +105,14 @@ data "template_file" "vm_config" { data "template_file" "pypi_sources_config" { template = file("${path.module}/pypi_sources_config.sh") vars = { - nexus_proxy_url = local.nexus_proxy_url + nexus_proxy_url = local.nexus_proxy_url[var.nexus_version] } } data "template_file" "apt_sources_config" { template = file("${path.module}/apt_sources_config.yml") vars = { - nexus_proxy_url = local.nexus_proxy_url + nexus_proxy_url = local.nexus_proxy_url[var.nexus_version] } } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/locals.tf index 4846e924d7..07d6ca0c62 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/locals.tf @@ -9,7 +9,11 @@ locals { vm_name = "linuxvm${local.short_service_id}" keyvault_name = lower("kv-${substr(local.workspace_resource_name_suffix, -20, -1)}") storage_name = lower(replace("stg${substr(local.workspace_resource_name_suffix, -8, -1)}", "-", "")) - nexus_proxy_url = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" + + nexus_proxy_url = { + "V1" = "https://nexus-${var.tre_id}.azurewebsites.net", + "V2" = "https://nexus-${var.tre_id}.${data.azurerm_resource_group.core.location}.cloudapp.azure.com" + } vm_size = { "2 CPU | 8GB RAM" = { value = "Standard_D2s_v5" }, "4 CPU | 16GB RAM" = { value = "Standard_D4s_v5" }, diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/variables.tf index 11b9a24565..3f3fd36c4d 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-dev-vm/terraform/variables.tf @@ -12,3 +12,4 @@ variable "image" {} variable "vm_size" {} variable "shared_storage_access" {} variable "shared_storage_name" {} +variable "nexus_version" {} diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index 7f8d072678..7488d5c54e 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,6 +1,6 @@ --- name: tre-workspace-base -version: 0.3.1 +version: 0.3.2 description: "A base Azure TRE workspace" registry: azuretre From 7693910cc6cbb921979fbd4599c43ff15e75c1a1 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 26 May 2022 13:05:28 +0000 Subject: [PATCH 137/142] Added docs for nexus_version --- .../configuring-shared-services.md | 18 +++++++++--------- .../installing-base-workspace.md | 3 ++- ...ling-workspace-service-and-user-resource.md | 8 ++++++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 71ca38095b..19b2973e39 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -2,15 +2,15 @@ ## Deploy/configure Nexus -If you're deploying a brand new environment you should deploy the VM-based service (read section `A`). If you wish to migrate from an existing App Service Nexus service to the VM-based service, first deploy the new service (section `A`) then proceed to section `B`. +If you're deploying a brand new environment you should deploy the VM-based (V2) service (read section `A`). If you wish to migrate from an existing App Service Nexus service (V1) to the VM-based service, first deploy the new service (section `A`) then proceed to section `B`. !!! info - The Makefile commands for deploying shared services temporarily target the App Service Nexus service so that existing environments won't have a new Nexus service deployed automatically by CICD and introduce breaking changes. The VM-based Nexus service will need to be deployed manually using the steps below and is required when deploying new Guacamole user resources of version `0.2.0` or higher. + The Makefile commands for deploying shared services temporarily target the V1 service so that existing environments won't have a new V2 Nexus service deployed automatically by CICD and introduce breaking changes. The V2 Nexus service will need to be deployed manually using the steps below. -### A. Deploy & configure Nexus service (hosted on VM) +### A. Deploy & configure V2 Nexus service (hosted on VM) !!! caution - Before deploying the VM-based Nexus service, you will need workspaces of version `0.2.14` or above due to a dependency on a DNS zone link for the workspace(s) to connect to the Nexus VM. + Before deploying the V2 Nexus service, you will need workspaces of version `0.3.2` or above due to a dependency on a DNS zone link for the workspace(s) to connect to the Nexus VM. Before deploying the Nexus shared service, you need to make sure that it will have access to a certificate to configure serving secure proxies. By default, the Nexus service will serve proxies from `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`, and thus it requires a certificate that validates ownership of this domain to use for SSL. @@ -83,13 +83,13 @@ You can optionally go to the Nexus web interface by visiting `https://nexus-{TRE Just bear in mind that if this service is redeployed any changes in the UI won't be persisted. If you wish to add new repositories or alter existing ones, use the JSON files within the `./nexus_repos_config` directory. -### B. Migrate from an existing Nexus service (hosted on App Service) +### B. Migrate from an existing V1 Nexus service (hosted on App Service) -Once you've created the new VM-based Nexus service by following section `A`, you can migrate from the old App Service Nexus service by following these steps: +Once you've created the new V2 (VM-based) Nexus service by following section `A`, you can migrate from the V1 Nexus service by following these steps: -1. Identify any existing Guacamole user resources that are using the old proxy URL (`https://nexus-{TRE_ID}.azurewebsites.net/`). These will be any VMs with bundle versions < `0.2.0`. +1. Identify any existing Guacamole user resources that are using the old proxy URL (`https://nexus-{TRE_ID}.azurewebsites.net/`). These will be any VMs with bundle versions < `0.3.2`. -1. These will need to be either **re-deployed** with the new template versions `0.2.0` or later (which target the new proxy URL format of `https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`), or manually have their proxy URLs updated by remoting into the VMs and updating the various configuration files of required package managers with the new URL. +1. These will need to be either **re-deployed** with the new template versions `0.3.2` or later and specifying an additional template parameter `"nexus_version"` with the value of `"V2"`, or manually have their proxy URLs updated by remoting into the VMs and updating the various configuration files of required package managers with the new URL (`https://nexus-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`). 1. For example, pip will need the `index`, `index-url` and `trusted-host` values in the global `pip.conf` file to be modified to use the new URL. @@ -97,7 +97,7 @@ Once you've created the new VM-based Nexus service by following section `A`, you ### Upgrade notes -The new Nexus shared service can be located in the `./templates/shared_services/sonatype-nexus-vm` directory, with the bundle name `tre-shared-service-sonatype-nexus`, which is now hosted using a VM to enable additional configuration required for proxying certain repositories. +The new V2 Nexus shared service can be located in the `./templates/shared_services/sonatype-nexus-vm` directory, with the bundle name `tre-shared-service-sonatype-nexus`, which is now hosted using a VM to enable additional configuration required for proxying certain repositories. This has been created as a separate service as the domain name exposed for proxies will be different to the one used by the original Nexus service and thus will break any user resources configured with the old proxy URL. diff --git a/docs/tre-admins/setup-instructions/installing-base-workspace.md b/docs/tre-admins/setup-instructions/installing-base-workspace.md index 82f74b1001..48cf20eacb 100644 --- a/docs/tre-admins/setup-instructions/installing-base-workspace.md +++ b/docs/tre-admins/setup-instructions/installing-base-workspace.md @@ -70,4 +70,5 @@ Workspace level operations can now be carried out using the workspace API, at `/ ## Next steps -* [Installing a workspace service](./installing-workspace-service-and-user-resource.md) +* [Configuring shared services](./configuring-shared-services.md) +* [Installing a workspace service & user resources](./installing-workspace-service-and-user-resource.md) diff --git a/docs/tre-admins/setup-instructions/installing-workspace-service-and-user-resource.md b/docs/tre-admins/setup-instructions/installing-workspace-service-and-user-resource.md index ab390b7ebc..a57ffae138 100644 --- a/docs/tre-admins/setup-instructions/installing-workspace-service-and-user-resource.md +++ b/docs/tre-admins/setup-instructions/installing-workspace-service-and-user-resource.md @@ -96,6 +96,9 @@ You can also follow the progress in Azure portal as various resources come up. Once the workspace service has been created, we can use the workspace API to create a user resource in our workspace. +!!! caution + Before deploying Guacamole user resources, you will want to make sure you have a Nexus shared service deployed in the workspace so that your VMs can access package repositories through a proxy (as they can't access public repositories directly). See [Configuring shared services](./configuring-shared-services.md). + 1. Navigate to the Swagger UI at `https:///api/workspaces//docs` . Where `` is the workspace ID of your workspace. 1. Click `Try it out` on the `POST` `/api/workspaces//workspace-services//user_resources` operation. Where `` and `` are the workspace ID of your workspace and workspace service ID of your workspace service. @@ -110,12 +113,13 @@ Once the workspace service has been created, we can use the workspace API to cre "properties": { "display_name": "My VM", "description": "Will be using this VM for my research", - "os_image": "Server 2019 Data Science VM" + "os_image": "Server 2019 Data Science VM", + "nexus_version": "V2" } } ``` - > Note: You can also specify "Windows 10" for a standard Windows 10 image + > Note: You can also specify "Windows 10" in "os_image" for a standard Windows 10 image. The "nexus_version" property also accepts "V1" if you have a V1 Nexus shared service deployed instead of the V2 service described in [Configuring shared services](./configuring-shared-services.md). The API will return an `operation` object with a `Location` header to query the operation status, as well as the `resourceId` and `resourcePath` properties to query the resource under creation. From dbf5735b24ebfc9f4a81fa9c19920d762820fb4f Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 26 May 2022 14:54:00 +0000 Subject: [PATCH 138/142] downgrade superlinter --- .github/workflows/build_validation_develop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index 8f46be0667..a875162408 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -48,7 +48,7 @@ jobs: - name: Lint code base # the slim image is 2GB smaller and we don't use the extra stuff # Moved this after the Terraform checks above due something similar to this issue: https://github.com/github/super-linter/issues/2433 - uses: github/super-linter/slim@v4.9.2 + uses: github/super-linter/slim@v4.9 env: VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: main From dd0ffd1b7abd39178bb4d051008b1301af6e8c2f Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 26 May 2022 15:52:45 +0000 Subject: [PATCH 139/142] revert superlinter to v4 --- .github/workflows/build_validation_develop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index a875162408..c633663ae1 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -48,7 +48,7 @@ jobs: - name: Lint code base # the slim image is 2GB smaller and we don't use the extra stuff # Moved this after the Terraform checks above due something similar to this issue: https://github.com/github/super-linter/issues/2433 - uses: github/super-linter/slim@v4.9 + uses: github/super-linter/slim@v4 env: VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: main From 25c08b6b25f2203b7be44cececef73e607f9f264 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 26 May 2022 16:04:56 +0000 Subject: [PATCH 140/142] Remove lint aws plugin block --- .github/linters/.tflint.hcl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/linters/.tflint.hcl b/.github/linters/.tflint.hcl index e48c4e5116..9ecdd2c68a 100644 --- a/.github/linters/.tflint.hcl +++ b/.github/linters/.tflint.hcl @@ -3,11 +3,6 @@ config { force = false } -# https://github.com/github/super-linter/issues/2954 -plugin "aws" { - enabled = false # Override: disable AWS -} - plugin "azurerm" { enabled = true } From 51bdbde1f18acdc0dbff794e4815e658c927ea40 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 26 May 2022 16:12:37 +0000 Subject: [PATCH 141/142] Use superlinter latest --- .github/workflows/build_validation_develop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index c633663ae1..78a172a7a2 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -48,7 +48,7 @@ jobs: - name: Lint code base # the slim image is 2GB smaller and we don't use the extra stuff # Moved this after the Terraform checks above due something similar to this issue: https://github.com/github/super-linter/issues/2433 - uses: github/super-linter/slim@v4 + uses: github/super-linter/slim@v4.9.3 env: VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: main From 76d509d93307aefd531c4d3435941ea06662dbb9 Mon Sep 17 00:00:00 2001 From: James Griffin Date: Thu, 26 May 2022 16:17:06 +0000 Subject: [PATCH 142/142] Manually set tflint path --- .github/workflows/build_validation_develop.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index 78a172a7a2..a811cace59 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -50,6 +50,9 @@ jobs: # Moved this after the Terraform checks above due something similar to this issue: https://github.com/github/super-linter/issues/2433 uses: github/super-linter/slim@v4.9.3 env: + # Until https://github.com/github/super-linter/commit/ec0662756da93f1e3aad4df049712df7d764d143 is released + # we need to set the correct plugin directory (which is incorrectly set to github/home/.tflint.d/plugins by default) + TFLINT_PLUGIN_DIR: "/root/.tflint.d/plugins" VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}