From 448502b02b3c523e47d5a14a41f5ab0be597cef2 Mon Sep 17 00:00:00 2001 From: anttorre Date: Thu, 7 Oct 2021 14:32:50 +0200 Subject: [PATCH 01/17] removed portal and management from apim --- src/core/README.md | 4 -- src/core/api/base_policy.tpl | 2 - src/core/apim.tf | 21 ---------- src/core/appgateway.tf | 63 +----------------------------- src/core/dns_public.tf | 18 --------- src/core/env/dev/terraform.tfvars | 2 - src/core/env/prod/terraform.tfvars | 2 - src/core/env/uat/terraform.tfvars | 2 - src/core/outputs.tf | 4 -- src/core/security.tf | 10 ----- src/core/variables.tf | 10 ----- 11 files changed, 2 insertions(+), 136 deletions(-) diff --git a/src/core/README.md b/src/core/README.md index 22dafcbf8..398c04890 100644 --- a/src/core/README.md +++ b/src/core/README.md @@ -55,8 +55,6 @@ | [azuread_service_principal.azdo_sp_tls_cert](https://registry.terraform.io/providers/hashicorp/azuread/2.3.0/docs/data-sources/service_principal) | data source | | [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/2.76.0/docs/data-sources/client_config) | data source | | [azurerm_key_vault_certificate.app_gw_platform](https://registry.terraform.io/providers/hashicorp/azurerm/2.76.0/docs/data-sources/key_vault_certificate) | data source | -| [azurerm_key_vault_certificate.management_platform](https://registry.terraform.io/providers/hashicorp/azurerm/2.76.0/docs/data-sources/key_vault_certificate) | data source | -| [azurerm_key_vault_certificate.portal_platform](https://registry.terraform.io/providers/hashicorp/azurerm/2.76.0/docs/data-sources/key_vault_certificate) | data source | | [azurerm_key_vault_secret.apim_publisher_email](https://registry.terraform.io/providers/hashicorp/azurerm/2.76.0/docs/data-sources/key_vault_secret) | data source | | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/2.76.0/docs/data-sources/subscription) | data source | @@ -67,8 +65,6 @@ | [apim\_publisher\_name](#input\_apim\_publisher\_name) | apim | `string` | n/a | yes | | [apim\_sku](#input\_apim\_sku) | n/a | `string` | n/a | yes | | [app\_gateway\_api\_certificate\_name](#input\_app\_gateway\_api\_certificate\_name) | Application gateway api certificate name on Key Vault | `string` | n/a | yes | -| [app\_gateway\_management\_certificate\_name](#input\_app\_gateway\_management\_certificate\_name) | Application gateway api management certificate name on Key Vault | `string` | n/a | yes | -| [app\_gateway\_portal\_certificate\_name](#input\_app\_gateway\_portal\_certificate\_name) | Application gateway developer portal certificate name on Key Vault | `string` | n/a | yes | | [cidr\_subnet\_appgateway](#input\_cidr\_subnet\_appgateway) | Application gateway address space. | `list(string)` | n/a | yes | | [cidr\_vnet](#input\_cidr\_vnet) | Virtual network address space. | `list(string)` | n/a | yes | | [cidr\_vnet\_integration](#input\_cidr\_vnet\_integration) | Virtual network to peer with sia subscription. It should host apim | `list(string)` | n/a | yes | diff --git a/src/core/api/base_policy.tpl b/src/core/api/base_policy.tpl index c9e15e4b7..568aaf825 100644 --- a/src/core/api/base_policy.tpl +++ b/src/core/api/base_policy.tpl @@ -2,8 +2,6 @@ - https://${portal-domain} - https://${management-api-domain} https://${apim-name}.developer.azure-api.net diff --git a/src/core/apim.tf b/src/core/apim.tf index 30868d317..c486fbd0c 100644 --- a/src/core/apim.tf +++ b/src/core/apim.tf @@ -19,11 +19,8 @@ resource "azurerm_resource_group" "rg_api" { locals { apim_cert_name_proxy_endpoint = format("%s-proxy-endpoint-cert", local.project) - portal_cert_name_proxy_endpoint = format("%s-proxy-endpoint-cert", "portal") api_domain = format("api.%s.%s", var.dns_zone_prefix, var.external_domain) - portal_domain = format("portal.%s.%s", var.dns_zone_prefix, var.external_domain) - management_domain = format("management.%s.%s", var.dns_zone_prefix, var.external_domain) } ########################### @@ -58,8 +55,6 @@ module "apim" { application_insights_instrumentation_key = azurerm_application_insights.application_insights.instrumentation_key xml_content = templatefile("./api/base_policy.tpl", { - portal-domain = local.portal_domain - management-api-domain = local.management_domain apim-name = format("%s-apim", local.project) }) @@ -80,22 +75,6 @@ resource "azurerm_api_management_custom_domain" "api_custom_domain" { data.azurerm_key_vault_certificate.app_gw_platform.version ) } - - developer_portal { - host_name = local.portal_domain - key_vault_id = trimsuffix( - data.azurerm_key_vault_certificate.portal_platform.secret_id, - data.azurerm_key_vault_certificate.portal_platform.version - ) - } - - management { - host_name = local.management_domain - key_vault_id = trimsuffix( - data.azurerm_key_vault_certificate.management_platform.secret_id, - data.azurerm_key_vault_certificate.management_platform.version - ) - } } ######### diff --git a/src/core/appgateway.tf b/src/core/appgateway.tf index 15d98c46c..9b7179049 100644 --- a/src/core/appgateway.tf +++ b/src/core/appgateway.tf @@ -60,24 +60,6 @@ module "app_gw" { probe = "/status-0123456789abcdef" probe_name = "probe-apim" } - - portal = { - protocol = "Https" - host = trim(azurerm_dns_a_record.dns_a_portal.fqdn, ".") - port = 443 - ip_addresses = module.apim.private_ip_addresses - probe = "/signin" - probe_name = "probe-portal" - } - - management = { - protocol = "Https" - host = trim(azurerm_dns_a_record.dns_a_management.fqdn, ".") - port = 443 - ip_addresses = module.apim.private_ip_addresses - probe = "/ServiceStatus" - probe_name = "probe-management" - } } ssl_profiles = [{ @@ -117,36 +99,6 @@ module "app_gw" { ) } } - - portal = { - protocol = "Https" - host = format("portal.%s.%s", var.dns_zone_prefix, var.external_domain) - port = 443 - ssl_profile_name = format("%s-ssl-profile", local.project) - - certificate = { - name = var.app_gateway_portal_certificate_name - id = trimsuffix( - data.azurerm_key_vault_certificate.portal_platform.secret_id, - data.azurerm_key_vault_certificate.portal_platform.version - ) - } - } - - management = { - protocol = "Https" - host = format("management.%s.%s", var.dns_zone_prefix, var.external_domain) - port = 443 - ssl_profile_name = format("%s-ssl-profile", local.project) - - certificate = { - name = var.app_gateway_management_certificate_name - id = trimsuffix( - data.azurerm_key_vault_certificate.management_platform.secret_id, - data.azurerm_key_vault_certificate.management_platform.version - ) - } - } } # maps listener to backend @@ -155,16 +107,6 @@ module "app_gw" { listener = "api" backend = "apim" } - - portal = { - listener = "portal" - backend = "portal" - } - - mangement = { - listener = "management" - backend = "management" - } } # TLS @@ -175,9 +117,8 @@ module "app_gw" { app_gateway_max_capacity = var.app_gateway_max_capacity # Logs - # todo enable - # sec_log_analytics_workspace_id = var.env_short == "p" ? data.azurerm_key_vault_secret.sec_workspace_id[0].value : null - # sec_storage_id = var.env_short == "p" ? data.azurerm_key_vault_secret.sec_storage_id[0].value : null + sec_log_analytics_workspace_id = var.env_short == "p" ? data.azurerm_key_vault_secret.sec_workspace_id[0].value : null + sec_storage_id = var.env_short == "p" ? data.azurerm_key_vault_secret.sec_storage_id[0].value : null tags = var.tags } diff --git a/src/core/dns_public.tf b/src/core/dns_public.tf index e75beca7c..ae2e89219 100644 --- a/src/core/dns_public.tf +++ b/src/core/dns_public.tf @@ -47,21 +47,3 @@ resource "azurerm_dns_a_record" "dns_a_api" { records = [azurerm_public_ip.appgateway_public_ip.ip_address] tags = var.tags } - -resource "azurerm_dns_a_record" "dns_a_portal" { - name = "portal" - zone_name = azurerm_dns_zone.selfcare_public[0].name - resource_group_name = azurerm_resource_group.rg_vnet.name - ttl = var.dns_default_ttl_sec - records = [azurerm_public_ip.appgateway_public_ip.ip_address] - tags = var.tags -} - -resource "azurerm_dns_a_record" "dns_a_management" { - name = "management" - zone_name = azurerm_dns_zone.selfcare_public[0].name - resource_group_name = azurerm_resource_group.rg_vnet.name - ttl = var.dns_default_ttl_sec - records = [azurerm_public_ip.appgateway_public_ip.ip_address] - tags = var.tags -} diff --git a/src/core/env/dev/terraform.tfvars b/src/core/env/dev/terraform.tfvars index 63818ad0f..d0060ecbc 100644 --- a/src/core/env/dev/terraform.tfvars +++ b/src/core/env/dev/terraform.tfvars @@ -37,8 +37,6 @@ apim_sku = "Developer_1" # app_gateway app_gateway_api_certificate_name = "api-dev-selfcare-pagopa-it" -app_gateway_portal_certificate_name = "portal-dev-selfcare-pagopa-it" -app_gateway_management_certificate_name = "management-dev-selfcare-pagopa-it" # postgresql prostgresql_enabled = true diff --git a/src/core/env/prod/terraform.tfvars b/src/core/env/prod/terraform.tfvars index 578b6594c..15d1d7ae5 100644 --- a/src/core/env/prod/terraform.tfvars +++ b/src/core/env/prod/terraform.tfvars @@ -39,8 +39,6 @@ apim_sku = "Developer_1" # todo change to Premium_1 before launch # app_gateway app_gateway_api_certificate_name = "api-selfcare-pagopa-it" -app_gateway_portal_certificate_name = "portal-selfcare-pagopa-it" -app_gateway_management_certificate_name = "management-selfcare-pagopa-it" app_gateway_min_capacity = 0 # todo change to at least 1 app_gateway_max_capacity = 2 diff --git a/src/core/env/uat/terraform.tfvars b/src/core/env/uat/terraform.tfvars index c85789e36..084449ce2 100644 --- a/src/core/env/uat/terraform.tfvars +++ b/src/core/env/uat/terraform.tfvars @@ -37,5 +37,3 @@ apim_sku = "Developer_1" # app_gateway app_gateway_api_certificate_name = "api-uat-selfcare-pagopa-it" -app_gateway_portal_certificate_name = "portal-uat-selfcare-pagopa-it" -app_gateway_management_certificate_name = "management-uat-selfcare-pagopa-it" diff --git a/src/core/outputs.tf b/src/core/outputs.tf index 9f67c0f52..58e8413be 100644 --- a/src/core/outputs.tf +++ b/src/core/outputs.tf @@ -13,7 +13,3 @@ output "vnet_integration_name" { output "vnet_integration_address_space" { value = module.vnet_integration.address_space } - -# output "azurerm_key_vault_certificate_management_platform" { -# value = data.azurerm_key_vault_certificate.management_platform -# } diff --git a/src/core/security.tf b/src/core/security.tf index 37e4e037d..bf975ee55 100644 --- a/src/core/security.tf +++ b/src/core/security.tf @@ -146,16 +146,6 @@ data "azurerm_key_vault_certificate" "app_gw_platform" { key_vault_id = module.key_vault.id } -data "azurerm_key_vault_certificate" "portal_platform" { - name = var.app_gateway_portal_certificate_name - key_vault_id = module.key_vault.id -} - -data "azurerm_key_vault_certificate" "management_platform" { - name = var.app_gateway_management_certificate_name - key_vault_id = module.key_vault.id -} - data "azurerm_key_vault_secret" "monitor_notification_slack_email" { name = "monitor-notification-slack-email" key_vault_id = module.key_vault.id diff --git a/src/core/variables.tf b/src/core/variables.tf index a76afa06e..784c97d00 100644 --- a/src/core/variables.tf +++ b/src/core/variables.tf @@ -162,16 +162,6 @@ variable "app_gateway_api_certificate_name" { description = "Application gateway api certificate name on Key Vault" } -variable "app_gateway_portal_certificate_name" { - type = string - description = "Application gateway developer portal certificate name on Key Vault" -} - -variable "app_gateway_management_certificate_name" { - type = string - description = "Application gateway api management certificate name on Key Vault" -} - # Scaling variable "app_gateway_min_capacity" { From 7a2adfb8661c0f53575d377f1067d3987a64a90b Mon Sep 17 00:00:00 2001 From: anttorre Date: Thu, 7 Oct 2021 15:50:39 +0200 Subject: [PATCH 02/17] cleaned dev vars --- src/core/env/dev/terraform.tfvars | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/core/env/dev/terraform.tfvars b/src/core/env/dev/terraform.tfvars index d0060ecbc..6c6baa032 100644 --- a/src/core/env/dev/terraform.tfvars +++ b/src/core/env/dev/terraform.tfvars @@ -36,17 +36,4 @@ apim_publisher_name = "pagoPA SelfCare DEV" apim_sku = "Developer_1" # app_gateway -app_gateway_api_certificate_name = "api-dev-selfcare-pagopa-it" - -# postgresql -prostgresql_enabled = true -postgresql_sku_name = "GP_Gen5_2" # todo fixme verify -postgresql_enable_replica = false -postgresql_public_network_access_enabled = true -postgresql_network_rules = { - ip_rules = [ - "0.0.0.0/0" - ] - # dblink - allow_access_to_azure_services = false -} +app_gateway_api_certificate_name = "api-dev-selfcare-pagopa-it" \ No newline at end of file From aa2454d1b69c7c4764f2dd775662fdf7678e302c Mon Sep 17 00:00:00 2001 From: anttorre Date: Thu, 7 Oct 2021 20:18:29 +0200 Subject: [PATCH 03/17] fixed certificate reference from keyvault --- src/core/apim.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/apim.tf b/src/core/apim.tf index c486fbd0c..d500a54ee 100644 --- a/src/core/apim.tf +++ b/src/core/apim.tf @@ -70,9 +70,10 @@ resource "azurerm_api_management_custom_domain" "api_custom_domain" { proxy { host_name = local.api_domain - key_vault_id = trimsuffix( + key_vault_id = replace( data.azurerm_key_vault_certificate.app_gw_platform.secret_id, - data.azurerm_key_vault_certificate.app_gw_platform.version + "/${data.azurerm_key_vault_certificate.app_gw_platform.version}", + "" ) } } From e322b6ef7085761c65823bd5b6677cdaa93644f2 Mon Sep 17 00:00:00 2001 From: anttorre Date: Thu, 7 Oct 2021 20:30:19 +0200 Subject: [PATCH 04/17] precommit run --- src/core/apim.tf | 10 +++++----- src/core/env/dev/terraform.tfvars | 2 +- src/core/env/prod/terraform.tfvars | 6 +++--- src/core/env/uat/terraform.tfvars | 2 +- src/core/monitor.tf | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/core/apim.tf b/src/core/apim.tf index d500a54ee..27f1d29f1 100644 --- a/src/core/apim.tf +++ b/src/core/apim.tf @@ -18,9 +18,9 @@ resource "azurerm_resource_group" "rg_api" { } locals { - apim_cert_name_proxy_endpoint = format("%s-proxy-endpoint-cert", local.project) + apim_cert_name_proxy_endpoint = format("%s-proxy-endpoint-cert", local.project) - api_domain = format("api.%s.%s", var.dns_zone_prefix, var.external_domain) + api_domain = format("api.%s.%s", var.dns_zone_prefix, var.external_domain) } ########################### @@ -55,7 +55,7 @@ module "apim" { application_insights_instrumentation_key = azurerm_application_insights.application_insights.instrumentation_key xml_content = templatefile("./api/base_policy.tpl", { - apim-name = format("%s-apim", local.project) + apim-name = format("%s-apim", local.project) }) tags = var.tags @@ -72,8 +72,8 @@ resource "azurerm_api_management_custom_domain" "api_custom_domain" { host_name = local.api_domain key_vault_id = replace( data.azurerm_key_vault_certificate.app_gw_platform.secret_id, - "/${data.azurerm_key_vault_certificate.app_gw_platform.version}", - "" + "/${data.azurerm_key_vault_certificate.app_gw_platform.version}", + "" ) } } diff --git a/src/core/env/dev/terraform.tfvars b/src/core/env/dev/terraform.tfvars index 6c6baa032..51591df39 100644 --- a/src/core/env/dev/terraform.tfvars +++ b/src/core/env/dev/terraform.tfvars @@ -36,4 +36,4 @@ apim_publisher_name = "pagoPA SelfCare DEV" apim_sku = "Developer_1" # app_gateway -app_gateway_api_certificate_name = "api-dev-selfcare-pagopa-it" \ No newline at end of file +app_gateway_api_certificate_name = "api-dev-selfcare-pagopa-it" diff --git a/src/core/env/prod/terraform.tfvars b/src/core/env/prod/terraform.tfvars index 15d1d7ae5..68e526b8e 100644 --- a/src/core/env/prod/terraform.tfvars +++ b/src/core/env/prod/terraform.tfvars @@ -38,9 +38,9 @@ apim_publisher_name = "pagoPA SelfCare PROD" apim_sku = "Developer_1" # todo change to Premium_1 before launch # app_gateway -app_gateway_api_certificate_name = "api-selfcare-pagopa-it" -app_gateway_min_capacity = 0 # todo change to at least 1 -app_gateway_max_capacity = 2 +app_gateway_api_certificate_name = "api-selfcare-pagopa-it" +app_gateway_min_capacity = 0 # todo change to at least 1 +app_gateway_max_capacity = 2 # todo change to Premium before launch # redis_sku_name = "Premium" diff --git a/src/core/env/uat/terraform.tfvars b/src/core/env/uat/terraform.tfvars index 084449ce2..31ce524aa 100644 --- a/src/core/env/uat/terraform.tfvars +++ b/src/core/env/uat/terraform.tfvars @@ -36,4 +36,4 @@ apim_publisher_name = "pagoPA SelfCare UAT" apim_sku = "Developer_1" # app_gateway -app_gateway_api_certificate_name = "api-uat-selfcare-pagopa-it" +app_gateway_api_certificate_name = "api-uat-selfcare-pagopa-it" diff --git a/src/core/monitor.tf b/src/core/monitor.tf index ea1b6af94..fb542a04f 100644 --- a/src/core/monitor.tf +++ b/src/core/monitor.tf @@ -61,4 +61,4 @@ resource "azurerm_monitor_action_group" "slack" { } tags = var.tags -} \ No newline at end of file +} From 31c5354fc85ba9cb1e0494d1e2e9891e7197abd7 Mon Sep 17 00:00:00 2001 From: anttorre Date: Fri, 8 Oct 2021 10:07:30 +0200 Subject: [PATCH 05/17] aks-infra --- src/core/aks.tf | 100 +++++++++++++++++++++++++++++ src/core/env/dev/terraform.tfvars | 1 + src/core/env/prod/terraform.tfvars | 1 + src/core/env/uat/terraform.tfvars | 1 + src/core/outputs.tf | 87 +++++++++++++++++++++++++ src/core/variables.tf | 84 ++++++++++++++++++++++++ 6 files changed, 274 insertions(+) create mode 100644 src/core/aks.tf diff --git a/src/core/aks.tf b/src/core/aks.tf new file mode 100644 index 000000000..40e7964c0 --- /dev/null +++ b/src/core/aks.tf @@ -0,0 +1,100 @@ +resource "azurerm_resource_group" "rg_aks" { + name = format("%s-aks-rg", local.project) + location = var.location + tags = var.tags +} + +module "aks" { + source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.58" + + name = format("%s-aks", local.project) + location = azurerm_resource_group.rg_aks.location + dns_prefix = format("%s-aks", local.project) + resource_group_name = azurerm_resource_group.rg_aks.name + availability_zones = var.aks_availability_zones + kubernetes_version = var.kubernetes_version + log_analytics_workspace_id = azurerm_log_analytics_workspace.log_analytics_workspace.id + + vm_size = var.aks_vm_size + node_count = var.aks_node_count + sku_tier = var.aks_sku_tier + + private_cluster_enabled = true + + rbac_enabled = true + aad_admin_group_ids = var.env_short == "d" ? [data.azuread_group.adgroup_admin.object_id, data.azuread_group.adgroup_developers.object_id] : [data.azuread_group.adgroup_admin.object_id] + + vnet_id = module.vnet.id + vnet_subnet_id = module.k8s_snet.id + + network_profile = { + docker_bridge_cidr = "172.17.0.1/16" + dns_service_ip = "10.2.0.10" + network_plugin = "azure" + network_policy = "azure" + outbound_type = "loadBalancer" + service_cidr = "10.2.0.0/16" + } + + metric_alerts = var.aks_metric_alerts + action = [ + { + action_group_id = azurerm_monitor_action_group.slack.id + webhook_properties = null + }, + { + action_group_id = azurerm_monitor_action_group.email.id + webhook_properties = null + } + ] + + alerts_enabled = var.aks_alerts_enabled + + outbound_ip_address_ids = azurerm_public_ip.aks_outbound.*.id + + tags = var.tags +} + +module "acr" { + source = "git::https://github.com/pagopa/azurerm.git//container_registry?ref=v1.0.58" + name = replace(format("%s-acr", local.project), "-", "") + resource_group_name = azurerm_resource_group.rg_aks.name + location = azurerm_resource_group.rg_aks.location + admin_enabled = false + + tags = var.tags +} + +# add the role to the identity the kubernetes cluster was assigned +resource "azurerm_role_assignment" "aks_to_acr" { + scope = module.acr.id + role_definition_name = "AcrPull" + principal_id = module.aks.kubelet_identity_id +} + +# k8s cluster subnet +module "k8s_snet" { + source = "git::https://github.com/pagopa/azurerm.git//subnet?ref=v1.0.58" + name = format("%s-k8s-snet", local.project) + address_prefixes = var.cidr_subnet_k8s + resource_group_name = azurerm_resource_group.rg_vnet.name + virtual_network_name = module.vnet.name + enforce_private_link_endpoint_network_policies = true + + service_endpoints = [ + "Microsoft.Web", + "Microsoft.Storage" + ] +} + +resource "azurerm_public_ip" "aks_outbound" { + count = var.aks_num_outbound_ips + + name = format("%s-aksoutbound-pip-%02d", local.project, count.index + 1) + resource_group_name = azurerm_resource_group.rg_vnet.name + location = azurerm_resource_group.rg_vnet.location + sku = "Standard" + allocation_method = "Static" + + tags = var.tags +} diff --git a/src/core/env/dev/terraform.tfvars b/src/core/env/dev/terraform.tfvars index 51591df39..c9dad319d 100644 --- a/src/core/env/dev/terraform.tfvars +++ b/src/core/env/dev/terraform.tfvars @@ -14,6 +14,7 @@ lock_enable = false # networking # main vnet cidr_vnet = ["10.1.0.0/16"] +cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] diff --git a/src/core/env/prod/terraform.tfvars b/src/core/env/prod/terraform.tfvars index 68e526b8e..0992f6f5c 100644 --- a/src/core/env/prod/terraform.tfvars +++ b/src/core/env/prod/terraform.tfvars @@ -14,6 +14,7 @@ lock_enable = true # networking # main vnet cidr_vnet = ["10.1.0.0/16"] +cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] # prod only diff --git a/src/core/env/uat/terraform.tfvars b/src/core/env/uat/terraform.tfvars index 31ce524aa..c95318ab3 100644 --- a/src/core/env/uat/terraform.tfvars +++ b/src/core/env/uat/terraform.tfvars @@ -14,6 +14,7 @@ lock_enable = true # networking # main vnet cidr_vnet = ["10.1.0.0/16"] +cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] diff --git a/src/core/outputs.tf b/src/core/outputs.tf index 58e8413be..ec08a8be2 100644 --- a/src/core/outputs.tf +++ b/src/core/outputs.tf @@ -13,3 +13,90 @@ output "vnet_integration_name" { output "vnet_integration_address_space" { value = module.vnet_integration.address_space } + +## AKS +output "aks_cluster_name" { + value = module.aks.name +} + +output "aks_fqdn" { + value = module.aks.fqdn +} + +output "aks_private_fqdn" { + value = module.aks.private_fqdn +} + +output "aks_client_certificate" { + value = module.aks.client_certificate +} + +output "aks_kube_config" { + value = module.aks.kube_config + sensitive = true +} + +output "aks_outbound_ips" { + value = azurerm_public_ip.aks_outbound.*.ip_address +} + +## key vault ## +output "key_vault_uri" { + value = module.key_vault.vault_uri +} + +output "key_vault_name" { + value = module.key_vault.name +} + +## Container registry ## +output "container_registry_login_server" { + value = module.acr.login_server +} + +output "container_registry_admin_username" { + value = module.acr.admin_username +} + +output "container_registry_admin_password" { + value = module.acr.admin_password + sensitive = true +} + +## Api management ## +output "apim_name" { + value = module.apim.name +} + +output "apim_private_ip_addresses" { + value = module.apim.private_ip_addresses +} + +output "apim_public_ip_addresses" { + value = module.apim.public_ip_addresses +} + +output "apim_gateway_url" { + value = format("https://%s", azurerm_api_management_custom_domain.api_custom_domain.proxy[0].host_name) +} + +output "apim_gateway_hostname" { + value = azurerm_api_management_custom_domain.api_custom_domain.proxy[0].host_name +} + +## Application gateway. +output "app_gateway_public_ip" { + value = azurerm_public_ip.appgateway_public_ip.ip_address +} + +output "app_gateway_fqdn" { + value = azurerm_public_ip.appgateway_public_ip.fqdn +} + +output "api_fqdn" { + value = azurerm_dns_a_record.dns_a_api.fqdn +} + +output "reverse_proxy_ip" { + value = var.reverse_proxy_ip +} diff --git a/src/core/variables.tf b/src/core/variables.tf index 784c97d00..4d472ccda 100644 --- a/src/core/variables.tf +++ b/src/core/variables.tf @@ -46,6 +46,90 @@ variable "tags" { } } +## AKS ## +variable "cidr_subnet_k8s" { + type = list(string) + description = "Subnet cluster kubernetes." +} + +variable "aks_availability_zones" { + type = list(number) + description = "A list of Availability Zones across which the Node Pool should be spread." + default = [] +} + +variable "aks_vm_size" { + type = string + default = "Standard_DS3_v2" + description = "The size of the AKS Virtual Machine in the Node Pool." +} + +variable "aks_node_count" { + type = number + description = "The initial number of the AKS nodes which should exist in this Node Pool." + default = 1 +} + +variable "kubernetes_version" { + type = string + default = null +} + +variable "aks_sku_tier" { + type = string + description = "The SKU Tier that should be used for this Kubernetes Cluster." + default = "Free" +} + +variable "reverse_proxy_ip" { + type = string + default = "127.0.0.1" + description = "AKS external ip. Also the ingress-nginx-controller external ip. Value known after installing the ingress controller." +} + +variable "aks_num_outbound_ips" { + type = number + default = 1 + description = "How many outbound ips allocate for AKS cluster" +} + +variable "aks_metric_alerts" { + default = {} + + description = < Date: Fri, 8 Oct 2021 10:08:34 +0200 Subject: [PATCH 06/17] precommit run --- src/core/env/dev/terraform.tfvars | 2 +- src/core/env/prod/terraform.tfvars | 2 +- src/core/env/uat/terraform.tfvars | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/env/dev/terraform.tfvars b/src/core/env/dev/terraform.tfvars index c9dad319d..982c09afc 100644 --- a/src/core/env/dev/terraform.tfvars +++ b/src/core/env/dev/terraform.tfvars @@ -14,7 +14,7 @@ lock_enable = false # networking # main vnet cidr_vnet = ["10.1.0.0/16"] -cidr_subnet_k8s = ["10.1.0.0/17"] +cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] diff --git a/src/core/env/prod/terraform.tfvars b/src/core/env/prod/terraform.tfvars index 0992f6f5c..63cad9c25 100644 --- a/src/core/env/prod/terraform.tfvars +++ b/src/core/env/prod/terraform.tfvars @@ -14,7 +14,7 @@ lock_enable = true # networking # main vnet cidr_vnet = ["10.1.0.0/16"] -cidr_subnet_k8s = ["10.1.0.0/17"] +cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] # prod only diff --git a/src/core/env/uat/terraform.tfvars b/src/core/env/uat/terraform.tfvars index c95318ab3..06d9c0c25 100644 --- a/src/core/env/uat/terraform.tfvars +++ b/src/core/env/uat/terraform.tfvars @@ -14,7 +14,7 @@ lock_enable = true # networking # main vnet cidr_vnet = ["10.1.0.0/16"] -cidr_subnet_k8s = ["10.1.0.0/17"] +cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] From 96424cc6669845822538b74392f5a5ed8f485978 Mon Sep 17 00:00:00 2001 From: anttorre Date: Fri, 8 Oct 2021 14:21:04 +0200 Subject: [PATCH 07/17] setting maxPods --- src/core/aks.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/aks.tf b/src/core/aks.tf index 40e7964c0..e8cdce51b 100644 --- a/src/core/aks.tf +++ b/src/core/aks.tf @@ -5,7 +5,7 @@ resource "azurerm_resource_group" "rg_aks" { } module "aks" { - source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.58" + source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.60" name = format("%s-aks", local.project) location = azurerm_resource_group.rg_aks.location @@ -18,6 +18,7 @@ module "aks" { vm_size = var.aks_vm_size node_count = var.aks_node_count sku_tier = var.aks_sku_tier + max_pods = var.env_short == "d" ? 100 : 30 private_cluster_enabled = true From 8c6ce48f1873ee336ae2d29e780346bb171cdbd2 Mon Sep 17 00:00:00 2001 From: anttorre Date: Fri, 8 Oct 2021 14:35:10 +0200 Subject: [PATCH 08/17] defining alert metrics --- src/core/env/dev/terraform.tfvars | 5 + src/core/env/prod/terraform.tfvars | 8 ++ src/core/env/uat/terraform.tfvars | 5 + src/core/variables.tf | 195 ++++++++++++++++++++++++++++- 4 files changed, 211 insertions(+), 2 deletions(-) diff --git a/src/core/env/dev/terraform.tfvars b/src/core/env/dev/terraform.tfvars index 982c09afc..cdd345694 100644 --- a/src/core/env/dev/terraform.tfvars +++ b/src/core/env/dev/terraform.tfvars @@ -38,3 +38,8 @@ apim_sku = "Developer_1" # app_gateway app_gateway_api_certificate_name = "api-dev-selfcare-pagopa-it" + +# aks +aks_alerts_enabled = false +# This is the k8s ingress controller ip. It must be in the aks subnet range. +reverse_proxy_ip = "10.1.0.250" diff --git a/src/core/env/prod/terraform.tfvars b/src/core/env/prod/terraform.tfvars index 63cad9c25..0b311f6bc 100644 --- a/src/core/env/prod/terraform.tfvars +++ b/src/core/env/prod/terraform.tfvars @@ -46,3 +46,11 @@ app_gateway_max_capacity = 2 # todo change to Premium before launch # redis_sku_name = "Premium" # redis_family = "P" + +# aks +# This is the k8s ingress controller ip. It must be in the aks subnet range. +reverse_proxy_ip = "10.1.0.250" +# aks_availability_zones = [1, 2, 3] # TODO to define and uncomment before release to prod +# aks_node_count = 6 # TODO to define and uncomment before release to prod +# aks_vm_size = "Standard_D8S_v3" # TODO to define and uncomment before release to prod +# aks_sku_tier = "Paid" # TODO to define and uncomment before release to prod diff --git a/src/core/env/uat/terraform.tfvars b/src/core/env/uat/terraform.tfvars index 06d9c0c25..13d876162 100644 --- a/src/core/env/uat/terraform.tfvars +++ b/src/core/env/uat/terraform.tfvars @@ -38,3 +38,8 @@ apim_sku = "Developer_1" # app_gateway app_gateway_api_certificate_name = "api-uat-selfcare-pagopa-it" + +# aks +aks_alerts_enabled = false +# This is the k8s ingress controller ip. It must be in the aks subnet range. +reverse_proxy_ip = "10.1.0.250" diff --git a/src/core/variables.tf b/src/core/variables.tf index 4d472ccda..381b72d62 100644 --- a/src/core/variables.tf +++ b/src/core/variables.tf @@ -94,8 +94,6 @@ variable "aks_num_outbound_ips" { } variable "aks_metric_alerts" { - default = {} - description = < Date: Fri, 8 Oct 2021 20:36:25 +0200 Subject: [PATCH 09/17] aks_node_count to 1 in PROD temporarily --- src/core/env/dev/terraform.tfvars | 2 +- src/core/env/prod/terraform.tfvars | 4 ++-- src/core/env/uat/terraform.tfvars | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/env/dev/terraform.tfvars b/src/core/env/dev/terraform.tfvars index 3b5a2c903..c66c0d027 100644 --- a/src/core/env/dev/terraform.tfvars +++ b/src/core/env/dev/terraform.tfvars @@ -17,7 +17,7 @@ cidr_vnet = ["10.1.0.0/16"] cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] -cidr_subnet_redis = ["10.1.132.0/24"] +cidr_subnet_redis = ["10.1.132.0/24"] # integration vnet # https://www.davidc.net/sites/default/subnets/subnets.html?network=10.230.7.0&mask=24&division=7.31 diff --git a/src/core/env/prod/terraform.tfvars b/src/core/env/prod/terraform.tfvars index 9c7c4d352..4e8701b35 100644 --- a/src/core/env/prod/terraform.tfvars +++ b/src/core/env/prod/terraform.tfvars @@ -17,7 +17,7 @@ cidr_vnet = ["10.1.0.0/16"] cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] -cidr_subnet_redis = ["10.1.132.0/24"] +cidr_subnet_redis = ["10.1.132.0/24"] # integration vnet # https://www.davidc.net/sites/default/subnets/subnets.html?network=10.230.7.0&mask=24&division=7.31 @@ -50,6 +50,6 @@ app_gateway_max_capacity = 2 # This is the k8s ingress controller ip. It must be in the aks subnet range. reverse_proxy_ip = "10.1.0.250" # aks_availability_zones = [1, 2, 3] # TODO to define and uncomment before release to prod -# aks_node_count = 6 # TODO to define and uncomment before release to prod +aks_node_count = 1 # TODO to define before release to prod # aks_vm_size = "Standard_D8S_v3" # TODO to define and uncomment before release to prod # aks_sku_tier = "Paid" # TODO to define and uncomment before release to prod diff --git a/src/core/env/uat/terraform.tfvars b/src/core/env/uat/terraform.tfvars index 6d6200a78..41090ffdf 100644 --- a/src/core/env/uat/terraform.tfvars +++ b/src/core/env/uat/terraform.tfvars @@ -17,7 +17,7 @@ cidr_vnet = ["10.1.0.0/16"] cidr_subnet_k8s = ["10.1.0.0/17"] cidr_subnet_appgateway = ["10.1.128.0/24"] cidr_subnet_azdoa = ["10.1.130.0/24"] -cidr_subnet_redis = ["10.1.132.0/24"] +cidr_subnet_redis = ["10.1.132.0/24"] # integration vnet # https://www.davidc.net/sites/default/subnets/subnets.html?network=10.230.7.0&mask=24&division=7.31 From 003762f3461665509f54c07a49565adc2fe3c2d0 Mon Sep 17 00:00:00 2001 From: anttorre Date: Mon, 11 Oct 2021 09:29:59 +0200 Subject: [PATCH 10/17] max_pods defined as variable --- src/core/aks.tf | 2 +- src/core/env/dev/terraform.tfvars | 1 + src/core/env/prod/terraform.tfvars | 1 + src/core/env/uat/terraform.tfvars | 1 + src/core/variables.tf | 6 ++++++ 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/core/aks.tf b/src/core/aks.tf index e8cdce51b..e5aee8be2 100644 --- a/src/core/aks.tf +++ b/src/core/aks.tf @@ -18,7 +18,7 @@ module "aks" { vm_size = var.aks_vm_size node_count = var.aks_node_count sku_tier = var.aks_sku_tier - max_pods = var.env_short == "d" ? 100 : 30 + max_pods = var.aks_max_pods private_cluster_enabled = true diff --git a/src/core/env/dev/terraform.tfvars b/src/core/env/dev/terraform.tfvars index c66c0d027..4c18ca6e1 100644 --- a/src/core/env/dev/terraform.tfvars +++ b/src/core/env/dev/terraform.tfvars @@ -44,3 +44,4 @@ app_gateway_api_certificate_name = "api-dev-selfcare-pagopa-it" aks_alerts_enabled = false # This is the k8s ingress controller ip. It must be in the aks subnet range. reverse_proxy_ip = "10.1.0.250" +aks_max_pods = 100 diff --git a/src/core/env/prod/terraform.tfvars b/src/core/env/prod/terraform.tfvars index 4e8701b35..38348e623 100644 --- a/src/core/env/prod/terraform.tfvars +++ b/src/core/env/prod/terraform.tfvars @@ -51,5 +51,6 @@ app_gateway_max_capacity = 2 reverse_proxy_ip = "10.1.0.250" # aks_availability_zones = [1, 2, 3] # TODO to define and uncomment before release to prod aks_node_count = 1 # TODO to define before release to prod +aks_max_pods = 100 # aks_vm_size = "Standard_D8S_v3" # TODO to define and uncomment before release to prod # aks_sku_tier = "Paid" # TODO to define and uncomment before release to prod diff --git a/src/core/env/uat/terraform.tfvars b/src/core/env/uat/terraform.tfvars index 41090ffdf..d1fadf733 100644 --- a/src/core/env/uat/terraform.tfvars +++ b/src/core/env/uat/terraform.tfvars @@ -44,3 +44,4 @@ app_gateway_api_certificate_name = "api-uat-selfcare-pagopa-it" aks_alerts_enabled = false # This is the k8s ingress controller ip. It must be in the aks subnet range. reverse_proxy_ip = "10.1.0.250" +aks_max_pods = 100 diff --git a/src/core/variables.tf b/src/core/variables.tf index 3c712c806..6d5d8a608 100644 --- a/src/core/variables.tf +++ b/src/core/variables.tf @@ -70,6 +70,12 @@ variable "aks_node_count" { default = 1 } +variable "aks_max_pods" { + type = number + description = "The maximum number of pods" + default = 100 +} + variable "kubernetes_version" { type = string default = null From 5e63fba4468436c054b8dd4c3aae3b941bc75ec4 Mon Sep 17 00:00:00 2001 From: anttorre Date: Mon, 11 Oct 2021 12:01:12 +0200 Subject: [PATCH 11/17] k8s base --- .gitignore | 1 + src/k8s/.terraform.lock.hcl | 78 +++++++ src/k8s/README.md | 204 ++++++++++++++++++ src/k8s/ingress.tf | 32 +++ src/k8s/ingress/loadbalancer.yaml.tpl | 5 + src/k8s/locals.tf | 6 + src/k8s/main.tf | 48 +++++ src/k8s/namespaces.tf | 11 + src/k8s/outputs.tf | 9 + src/k8s/rbac.tf | 176 +++++++++++++++ src/k8s/scripts/base_64_decode.sh | 12 ++ src/k8s/scripts/restart-pods.sh | 18 ++ src/k8s/scripts/setup.sh | 46 ++++ src/k8s/scripts/ssh-port-forward.sh | 36 ++++ src/k8s/secrets.tf | 10 + src/k8s/serviceaccounts.tf | 79 +++++++ .../subscriptions/DEV-SelfCare/backend.ini | 2 + .../DEV-SelfCare/terraform.tfvars | 6 + .../subscriptions/PROD-SelfCare/backend.ini | 2 + .../PROD-SelfCare/terraform.tfvars | 6 + .../subscriptions/UAT-SelfCare/backend.ini | 2 + .../UAT-SelfCare/terraform.tfvars | 6 + src/k8s/terraform.sh | 63 ++++++ src/k8s/variables.tf | 56 +++++ 24 files changed, 914 insertions(+) create mode 100644 src/k8s/.terraform.lock.hcl create mode 100644 src/k8s/README.md create mode 100644 src/k8s/ingress.tf create mode 100644 src/k8s/ingress/loadbalancer.yaml.tpl create mode 100644 src/k8s/locals.tf create mode 100644 src/k8s/main.tf create mode 100644 src/k8s/namespaces.tf create mode 100644 src/k8s/outputs.tf create mode 100644 src/k8s/rbac.tf create mode 100644 src/k8s/scripts/base_64_decode.sh create mode 100644 src/k8s/scripts/restart-pods.sh create mode 100644 src/k8s/scripts/setup.sh create mode 100644 src/k8s/scripts/ssh-port-forward.sh create mode 100644 src/k8s/secrets.tf create mode 100644 src/k8s/serviceaccounts.tf create mode 100644 src/k8s/subscriptions/DEV-SelfCare/backend.ini create mode 100644 src/k8s/subscriptions/DEV-SelfCare/terraform.tfvars create mode 100644 src/k8s/subscriptions/PROD-SelfCare/backend.ini create mode 100644 src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars create mode 100644 src/k8s/subscriptions/UAT-SelfCare/backend.ini create mode 100644 src/k8s/subscriptions/UAT-SelfCare/terraform.tfvars create mode 100644 src/k8s/terraform.sh create mode 100644 src/k8s/variables.tf diff --git a/.gitignore b/.gitignore index 5aadc243c..67d29c2d3 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ __TMP .metals/ __azurite_* /.idea +.bastianhost.ini diff --git a/src/k8s/.terraform.lock.hcl b/src/k8s/.terraform.lock.hcl new file mode 100644 index 000000000..3fdc33c40 --- /dev/null +++ b/src/k8s/.terraform.lock.hcl @@ -0,0 +1,78 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.5.0" + constraints = "2.5.0" + hashes = [ + "h1:Er35+K+GSrfZEJId/OqCWvOUa0idXQYyTrA2+I9KfI4=", + "zh:08e0ae5f1fde389a3cb9b32d3910fd0fe7cb6d361cf1133a22e803b7a7e66b8f", + "zh:093e70b0b4245605b6798be089defe385ac20e3a7f8aea64a7095bd4f762c5e9", + "zh:1fab548430864022308cb16b2fd9eebc993e63c1572aedd3ec9f81a2ebdc9e38", + "zh:5cc657d824b21f430a2c37d52c8a9a3ab06fdb3039a10eccd427bd5a6917ace1", + "zh:7863ab17f8cb12154d356d513e375772017904e6a7626ebb8e39210730afff6a", + "zh:7e53f8baa5a9279e4e7ed8533955f0e06bcbc8477fcb6a9bb22c10d5fc5c4d11", + "zh:91ed1dce045b6714cd8d1931e50347d1ef8ce5fc614229acc28527b0407a344b", + "zh:afce857b4eeb53f932ab0324e2f9ffb5014ece10e45dbfc5b7f09b9769123d90", + "zh:b8a81586ed314bdcfed05c5480a4135d4e251be2c41e8ba8026c669d951cb459", + "zh:db06bdf8d2d825399809e85a6b8be291e34416cda95e116e135f2ae605f7d2a0", + "zh:f5e03cf80fe30e0b1604ec0330ff2f2937acd512ae1a126684d6b61da2acb126", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "2.79.1" + constraints = "2.79.1" + hashes = [ + "h1:jPlACSoNUhzN7FpvgK8pze5IqtVt7Xw/0+kdzQmoIDI=", + "zh:120da7d501adb34c5600a09847e483e953cf45badb76c2e213f11beba5856bee", + "zh:2e1fb026cc2eb76aba2129661a9bdf0e4014668c2ea045208148b505aa708f6f", + "zh:3541d4daac9c07c6ffbb2f266c731efa32e08c3b72bd9b5454d368f418f3dd1c", + "zh:3efe2ee3cb51820d18ed5f3456f3ef8dcdfeb90d79790a1a1f8f3a8c2430e8a0", + "zh:46c1cc27031138f41e0bb4bee16df10431c963f23d0716583b7e66cf9be1b58a", + "zh:61125f3af098955320409dcb801a239059e3062937c9eded80cd2296bccabc3c", + "zh:681fc0457852db69dd1ce142c830e849f7d4293f7c6ab295bd493a1f7fb68133", + "zh:aa1e101f425f89a672e9821d11518fb93450d117ce6588852e87af369bdefc66", + "zh:be8120f98fcacaad9ae88986ea8018d715b03c00ff3fc23954af9c8be0c4e4f1", + "zh:d6c957b6fb43810a48c39ee907bdef2306b577a50df8cc231a6cd71650fb7009", + "zh:df243c69f4823935f34c2eb0a46f83050e2d78026bf953eacc580710dc4c1e40", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.2.0" + constraints = "~> 2.2.0" + hashes = [ + "h1:lFm6HwNEXgXT50K1jE7wnNaBLHTAt04KE5tjWQcJOMg=", + "zh:01341dd1e9cc7e7f6999e11e7473bcdca2dd72dd27f91beed1f4fb599a15dfba", + "zh:20e86c9eccd3a81ef5ac243af31b61fc4d2d679437384bd0870e92fa1b3ed6c9", + "zh:22a71127c5dbea4f62edb5bcf00b5c163de04aa19d45a7a1f621f973ffd09d20", + "zh:28ab7c84a5f8ed82fc520668db93d650571ddf59d98845cb18a1fa1a7888efc0", + "zh:3985a30929ad8fdc6b94f0e1cbd62a63db75ee961b8ba7db1cf4bfd29e8009ff", + "zh:477d92e26ba0c906087a5dd827ac3917dad7d5af770ee0ab4b08d0f273150586", + "zh:750928ec5ef54b2090bd6a6d8a19630a8712bbbccc0429251e88ccd361c1d3c0", + "zh:a615841fd90094bddc1269127e501fa60453c441b9548ff73752fe14efc38ed0", + "zh:e762aca7883374fa255efba50f5bdf791fece7d61e3920e593fb1a2cbb598981", + "zh:f76f372ead52948ca53610b371cb80c80ebcf058ef0a5c0ce9f0ce38dcc9a8eb", + "zh:fa36fe93ed977f4478cc6547ec3c45c28e56f10632e85446b0c3d71449f8c4bb", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.3.2" + constraints = "~> 2.3.2" + hashes = [ + "h1:D8HWX3vouTPI3Jicq43xOQyoYWtSsVua92cBVrJ3ZMs=", + "zh:10f71c170be13538374a4b9553fcb3d98a6036bcd1ca5901877773116c3f828e", + "zh:11d2230e531b7480317e988207a73cb67b332f225b0892304983b19b6014ebe0", + "zh:3317387a9a6cc27fd7536b8f3cad4b8a9285e9461f125c5a15d192cef3281856", + "zh:458a9858362900fbe97e00432ae8a5bef212a4dacf97a57ede7534c164730da4", + "zh:50ea297007d9fe53e5411577f87a4b13f3877ce732089b42f938430e6aadff0d", + "zh:56705c959e4cbea3b115782d04c62c68ac75128c5c44ee7aa4043df253ffbfe3", + "zh:7eb3722f7f036e224824470c3e0d941f1f268fcd5fa2f8203e0eee425d0e1484", + "zh:9f408a6df4d74089e6ce18f9206b06b8107ddb57e2bc9b958a6b7dc352c62980", + "zh:aadd25ccc3021040808feb2645779962f638766eb583f586806e59f24dde81bb", + "zh:b101c3456e4309b09aab129b0118561178c92cb4be5d96dec553189c3084dca1", + "zh:ec08478573b4953764099fbfd670fae81dc24b60e467fb3b023e6fab50b70a9e", + ] +} diff --git a/src/k8s/README.md b/src/k8s/README.md new file mode 100644 index 000000000..7c8b41a41 --- /dev/null +++ b/src/k8s/README.md @@ -0,0 +1,204 @@ +# kubernetes-infrastructure + +This is a kubernetes infrastructure configuration. + +## Requirements + +### 1. terraform + +In order to manage the suitable version of terraform it is strongly recommended to install the following tool: + +- [tfenv](https://github.com/tfutils/tfenv): **Terraform** version manager inspired by rbenv. + +Once these tools have been installed, install the terraform version shown in: + +- .terraform-version + +After installation install terraform: + +```sh +tfenv install +``` + +### 2. Azure CLI + +In order to authenticate to Azure portal and manage terraform state it's necessary to install and login to Azure subscription. + +- [Azure CLI](https://docs.microsoft.com/it-it/cli/azure/install-azure-cli) + +After installation login to Azure: + +```sh +az login +``` + +### 3. kubectl + +In order to run commands against Kubernetes clusters it's necessary to install kubectl. + +- [kubectl](https://kubernetes.io/docs/tasks/tools/) + +### 4. helm + +In order to use Helm package manager for Kubernetes it's necessary to install helm. + +- [helm](https://helm.sh/docs/helm/helm_install/) + +### 5. Access to bastian host (jumpbox) + +We deploy a kubernetes in private mode so it is not public accessible. +We use an SSH connection to a bastian host started on demand (jumpbox). + +```sh +## ~/.ssh/config file configuration +# Change project_aks_env_user, user and bastian_host_env_ip with correct values +# Ask to an Azure Administrator the id_rsa_project_aks_env_user private key +Host project_aks_env_user + AddKeysToAgent yes + UseKeychain yes + HostName bastian_host_env_ip + User user + IdentityFile ~/.ssh/id_rsa_project_aks_env_user +``` + +```sh +# set rw permission to id_rsa_project_aks_env_user key only for current user +chmod 600 ~/.ssh/id_rsa_project_aks_env_user +ssh-add ~/.ssh/id_rsa_project_aks_env_user +# if nedded, restart ssh-agent +eval "$(ssh-agent -s)" +``` + +## Terraform modules + +As PagoPA we build our standard Terraform modules, check available modules: + +- [PagoPA Terraform modules](https://github.com/search?q=topic%3Aterraform-modules+org%3Apagopa&type=repositories) + +## Setup configuration + +Before first use we need to run a setup script to configure `.bastianhost.ini` and download kube config. + +```sh +bash scripts/setup.sh ENV-PROJECT + +# example for SelfCare project in DEV environment +bash scripts/setup.sh DEV-SelfCare +``` + +## Apply changes + +To apply changes use `terraform.sh` script as follow: + +```sh +bash terraform.sh apply|plan|destroy ENV-PROJECT + +# example to apply configuration for SelfCare project in DEV environment +bash terraform.sh apply DEV-SelfCare +``` + +## Terraform lock.hcl + +We have both developers who work with your Terraform configuration on their Linux, macOS or Windows workstations and automated systems that apply the configuration while running on Linux. +https://www.terraform.io/docs/cli/commands/providers/lock.html#specifying-target-platforms + +So we need to specify this in terraform lock providers: + +```sh +terraform init + +rm .terraform.lock.hcl + +terraform providers lock \ + -platform=windows_amd64 \ + -platform=darwin_amd64 \ + -platform=linux_amd64 +``` + +## Precommit checks + +Check your code before commit. + +https://github.com/antonbabenko/pre-commit-terraform#how-to-install + +```sh +pre-commit run -a +``` + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >=0.15.3 | +| [azuread](#requirement\_azuread) | = 1.6.0 | +| [azurerm](#requirement\_azurerm) | ~> 2.60.0 | +| [helm](#requirement\_helm) | ~> 2.1.2 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.3.2 | + +## Providers + +| Name | Version | +|------|---------| +| [azuread](#provider\_azuread) | 1.6.0 | +| [azurerm](#provider\_azurerm) | 2.60.0 | +| [helm](#provider\_helm) | 2.1.2 | +| [kubernetes](#provider\_kubernetes) | 2.3.2 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [key\_vault\_secrets\_query](#module\_key\_vault\_secrets\_query) | git::https://github.com/pagopa/azurerm.git//key_vault_secrets_query | v1.0.58 | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_key_vault_secret.azure_devops_sa_cacrt](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.azure_devops_sa_token](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [helm_release.ingress](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_cluster_role.cluster_deployer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | +| [kubernetes_cluster_role.view_extra](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | +| [kubernetes_cluster_role_binding.edit_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | +| [kubernetes_cluster_role_binding.view_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | +| [kubernetes_cluster_role_binding.view_extra_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | +| [kubernetes_namespace.selc](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_namespace.ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_role_binding.deployer_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource | +| [kubernetes_secret.azure-storage](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | +| [kubernetes_secret.selc-application-insights](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | +| [kubernetes_service_account.azure_devops](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource | +| [azuread_group.adgroup_contributors](https://registry.terraform.io/providers/hashicorp/azuread/1.6.0/docs/data-sources/group) | data source | +| [azuread_group.adgroup_externals](https://registry.terraform.io/providers/hashicorp/azuread/1.6.0/docs/data-sources/group) | data source | +| [azuread_group.adgroup_security](https://registry.terraform.io/providers/hashicorp/azuread/1.6.0/docs/data-sources/group) | data source | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | +| [kubernetes_secret.azure_devops_secret](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/data-sources/secret) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [env](#input\_env) | n/a | `string` | n/a | yes | +| [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | +| [ingress\_load\_balancer\_ip](#input\_ingress\_load\_balancer\_ip) | n/a | `string` | n/a | yes | +| [ingress\_replica\_count](#input\_ingress\_replica\_count) | n/a | `string` | n/a | yes | +| [k8s\_apiserver\_host](#input\_k8s\_apiserver\_host) | n/a | `string` | n/a | yes | +| [rbac\_namespaces](#input\_rbac\_namespaces) | n/a | `list(string)` | n/a | yes | +| [default\_service\_port](#input\_default\_service\_port) | n/a | `number` | `8080` | no | +| [event\_hub\_port](#input\_event\_hub\_port) | n/a | `number` | `9093` | no | +| [k8s\_apiserver\_insecure](#input\_k8s\_apiserver\_insecure) | n/a | `bool` | `false` | no | +| [k8s\_apiserver\_port](#input\_k8s\_apiserver\_port) | n/a | `number` | `443` | no | +| [k8s\_kube\_config\_path](#input\_k8s\_kube\_config\_path) | n/a | `string` | `"~/.kube/config"` | no | +| [location](#input\_location) | n/a | `string` | `"westeurope"` | no | +| [prefix](#input\_prefix) | n/a | `string` | `"selc"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [azure\_devops\_sa\_cacrt](#output\_azure\_devops\_sa\_cacrt) | n/a | +| [azure\_devops\_sa\_token](#output\_azure\_devops\_sa\_token) | n/a | + diff --git a/src/k8s/ingress.tf b/src/k8s/ingress.tf new file mode 100644 index 000000000..ff39b68c3 --- /dev/null +++ b/src/k8s/ingress.tf @@ -0,0 +1,32 @@ +# from Microsoft docs https://docs.microsoft.com/it-it/azure/aks/ingress-internal-ip +resource "helm_release" "ingress" { + name = "nginx-ingress" + repository = "https://kubernetes.github.io/ingress-nginx" + chart = "ingress-nginx" + version = "3.31.0" + namespace = kubernetes_namespace.ingress.metadata[0].name + + values = [ + "${templatefile("${path.module}/ingress/loadbalancer.yaml.tpl", { load_balancer_ip = var.ingress_load_balancer_ip })}" + ] + + set { + name = "controller.replicaCount" + value = var.ingress_replica_count + } + + set { + name = "controller.nodeSelector.beta\\.kubernetes\\.io/os" + value = "linux" + } + + set { + name = "defaultBackend.nodeSelector.beta\\.kubernetes\\.io/os" + value = "linux" + } + + set { + name = "controller.admissionWebhooks.patch.nodeSelector.beta\\.kubernetes\\.io/os" + value = "linux" + } +} diff --git a/src/k8s/ingress/loadbalancer.yaml.tpl b/src/k8s/ingress/loadbalancer.yaml.tpl new file mode 100644 index 000000000..f00cb77ca --- /dev/null +++ b/src/k8s/ingress/loadbalancer.yaml.tpl @@ -0,0 +1,5 @@ +controller: + service: + loadBalancerIP: ${load_balancer_ip} + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" diff --git a/src/k8s/locals.tf b/src/k8s/locals.tf new file mode 100644 index 000000000..73e58dfa2 --- /dev/null +++ b/src/k8s/locals.tf @@ -0,0 +1,6 @@ +locals { + project = format("%s-%s", var.prefix, var.env_short) + key_vault_name = format("%s-kv", local.project) + key_vault_resource_group = format("%s-sec-rg", local.project) + key_vault_id = "${data.azurerm_subscription.current.id}/resourceGroups/${local.key_vault_resource_group}/providers/Microsoft.KeyVault/vaults/${local.key_vault_name}" +} diff --git a/src/k8s/main.tf b/src/k8s/main.tf new file mode 100644 index 000000000..48a55cc6e --- /dev/null +++ b/src/k8s/main.tf @@ -0,0 +1,48 @@ +terraform { + required_version = ">=0.15.3" + + backend "azurerm" { + container_name = "k8sstate" + key = "terraform.tfstate" + } + + required_providers { + azurerm = { + version = "= 2.79.1" + } + azuread = { + source = "hashicorp/azuread" + version = "= 2.5.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.3.2" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.2.0" + } + } +} + +provider "kubernetes" { + host = "https://${var.k8s_apiserver_host}:${var.k8s_apiserver_port}" + insecure = var.k8s_apiserver_insecure + config_path = var.k8s_kube_config_path +} + +provider "helm" { + kubernetes { + host = "https://${var.k8s_apiserver_host}:${var.k8s_apiserver_port}" + insecure = var.k8s_apiserver_insecure + config_path = var.k8s_kube_config_path + } +} + +provider "azurerm" { + features {} +} + +data "azurerm_subscription" "current" {} + +data "azurerm_client_config" "current" {} diff --git a/src/k8s/namespaces.tf b/src/k8s/namespaces.tf new file mode 100644 index 000000000..446a61c3a --- /dev/null +++ b/src/k8s/namespaces.tf @@ -0,0 +1,11 @@ +resource "kubernetes_namespace" "ingress" { + metadata { + name = "ingress" + } +} + +resource "kubernetes_namespace" "selc" { + metadata { + name = "selc" + } +} diff --git a/src/k8s/outputs.tf b/src/k8s/outputs.tf new file mode 100644 index 000000000..35d297504 --- /dev/null +++ b/src/k8s/outputs.tf @@ -0,0 +1,9 @@ +output "azure_devops_sa_token" { + value = data.kubernetes_secret.azure_devops_secret.binary_data["token"] + sensitive = true +} + +output "azure_devops_sa_cacrt" { + value = data.kubernetes_secret.azure_devops_secret.binary_data["ca.crt"] + sensitive = true +} diff --git a/src/k8s/rbac.tf b/src/k8s/rbac.tf new file mode 100644 index 000000000..840cae54d --- /dev/null +++ b/src/k8s/rbac.tf @@ -0,0 +1,176 @@ +data "azuread_group" "adgroup_externals" { + display_name = format("%s-adgroup-externals", local.project) +} + +data "azuread_group" "adgroup_developers" { + display_name = format("%s-adgroup-developers", local.project) +} + +data "azuread_group" "adgroup_security" { + display_name = format("%s-adgroup-security", local.project) +} + +data "azuread_group" "adgroup_operations" { + display_name = format("%s-adgroup-operations", local.project) +} + +data "azuread_group" "adgroup_technical_project_managers" { + display_name = format("%s-adgroup-technical-project-managers", local.project) +} + +resource "kubernetes_cluster_role" "view_extra" { + metadata { + name = "view-extra" + } + + dynamic "rule" { + for_each = var.env_short == "d" ? [""] : [] + + content { + api_groups = [""] + resources = ["pods/attach", "pods/exec", "pods/portforward", "pods/proxy", "secrets", "services/proxy"] + verbs = ["get", "list", "watch"] + } + } + + dynamic "rule" { + for_each = var.env_short == "d" ? [""] : [] + content { + api_groups = [""] + resources = ["pods/attach", "pods/exec", "pods/portforward", "pods/proxy"] + verbs = ["create", "delete", "deletecollection", "patch", "update"] + } + } +} + +resource "kubernetes_cluster_role_binding" "view_extra_binding" { + metadata { + name = "view-extra-binding" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.view_extra.metadata[0].name + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_security.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_developers.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_externals.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_operations.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_technical_project_managers.object_id + namespace = "kube-system" + } +} + +resource "kubernetes_cluster_role" "edit_extra" { + metadata { + name = "edit-extra" + } + + rule { + api_groups = ["rbac.authorization.k8s.io"] + resources = ["*"] + verbs = ["get", "list"] + } +} + +resource "kubernetes_cluster_role_binding" "edit_extra_binding" { + metadata { + name = "edit-extra-binding" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.edit_extra.metadata[0].name + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_developers.object_id + namespace = "kube-system" + } +} + +resource "kubernetes_cluster_role_binding" "edit_binding" { + metadata { + name = "edit-binding" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "edit" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_developers.object_id + namespace = "kube-system" + } +} + +resource "kubernetes_cluster_role_binding" "view_binding" { + metadata { + name = "view-binding" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "view" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_developers.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_security.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_externals.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_operations.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_technical_project_managers.object_id + namespace = "kube-system" + } +} diff --git a/src/k8s/scripts/base_64_decode.sh b/src/k8s/scripts/base_64_decode.sh new file mode 100644 index 000000000..e6d7996ce --- /dev/null +++ b/src/k8s/scripts/base_64_decode.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ $# -lt 1 ]; then + echo 1>&2 "$0: missed string to decode." + exit 2 +fi + +str=$1 + +echo Encoding $str +echo "" +echo "$str" | base64 --decode diff --git a/src/k8s/scripts/restart-pods.sh b/src/k8s/scripts/restart-pods.sh new file mode 100644 index 000000000..e8e731b94 --- /dev/null +++ b/src/k8s/scripts/restart-pods.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ $# -lt 1 ]; then + echo 1>&2 "$0: missed namespace!! usage: $0 " + exit 2 +elif [ $# -gt 1 ]; then + echo 1>&2 "$0: too many arguments" + exit 2 +fi + +namespace=$1 +waitfor=3s + +deploys=`kubectl -n $namespace get deployments | tail -n +2 | cut -d ' ' -f 1` +for deploy in $deploys; do + kubectl -n $1 rollout restart deployments/$deploy + sleep $waitfor +done diff --git a/src/k8s/scripts/setup.sh b/src/k8s/scripts/setup.sh new file mode 100644 index 000000000..9c83a53c3 --- /dev/null +++ b/src/k8s/scripts/setup.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# +# Setup configuration relative to a given subscription +# Subscription are defined in ./subscription +# Usage: +# ./setup.sh ENV-SelfCare +# +# ./setup.sh DEV-SelfCare +# ./setup.sh UAT-SelfCare +# ./setup.sh PROD-SelfCare + +BASHDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +WORKDIR="${BASHDIR//scripts/}" + +set -e + +SUBSCRIPTION=$1 + +if [ -z "${SUBSCRIPTION}" ]; then + printf "\e[1;31mYou must provide a subscription as first argument.\n" + exit 1 +fi + +if [ ! -d "${WORKDIR}/subscriptions/${SUBSCRIPTION}" ]; then + printf "\e[1;31mYou must provide a subscription for which a variable file is defined. You provided: '%s'.\n" "${SUBSCRIPTION}" > /dev/stderr + exit 1 +fi + +az account set -s "${SUBSCRIPTION}" + +aks_name=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:name}") +aks_resource_group_name=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:resourceGroup}") +aks_private_fqdn=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:privateFqdn}") + +rm -rf "${HOME}/.kube/config-${aks_name}" +az aks get-credentials -g "${aks_resource_group_name}" -n "${aks_name}" --subscription "${SUBSCRIPTION}" --file "~/.kube/config-${aks_name}" +az aks get-credentials -g "${aks_resource_group_name}" -n "${aks_name}" --subscription "${SUBSCRIPTION}" --overwrite-existing +echo "aks_private_fqdn=${aks_private_fqdn}" >> "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" +echo "kube_config_path=~/.kube/config-${aks_name}" >> "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" + +# with AAD auth enabled we need to authenticate the machine on the first setup +echo "Follow Microsoft sign in steps. kubectl get pods command will fail but it's the expected behavior" +kubectl --kubeconfig="${HOME}/.kube/config-${aks_name}" get pods +kubectl config use-context "${aks_name}" +kubectl get pods diff --git a/src/k8s/scripts/ssh-port-forward.sh b/src/k8s/scripts/ssh-port-forward.sh new file mode 100644 index 000000000..6aa47ed4d --- /dev/null +++ b/src/k8s/scripts/ssh-port-forward.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# From https://dev.to/jaysonsantos/using-terraform-s-remote-exec-provider-with-aws-ssm-5po + +set -ex +test -n "$DESTINATION_IP" || (echo missing DESTINATION_IP; exit 1) +test -n "$USERNAME" || (echo missing USERNAME; exit 1) +test -n "$RANDOM_PORT" || (echo missing RANDOM_PORT; exit 1) +test -n "$TARGET" || (echo missing TARGET; exit 1) + +set +e + +cleanup() { + cat log.txt + rm -rf log.txt + exit $! +} + +for try in {0..5}; do + echo "Trying to port forward retry #$try" + # The following command MUST NOT print to the stdio otherwise it will just + # inherit the pipe from the parent process and will hold terraform's lock + ssh -f -o StrictHostKeyChecking=no \ + -o ControlMaster=no \ + "$USERNAME@$DESTINATION_IP" \ + -L "127.0.0.1:$RANDOM_PORT:$TARGET" \ + sleep 15m &> log.txt # This is the special ingredient! + success="$?" + if [ "$success" -eq 0 ]; then + cleanup 0 + fi + sleep 5s +done + +echo "Failed to start a port forwarding session" +cleanup 1 diff --git a/src/k8s/secrets.tf b/src/k8s/secrets.tf new file mode 100644 index 000000000..88561cf75 --- /dev/null +++ b/src/k8s/secrets.tf @@ -0,0 +1,10 @@ +module "key_vault_secrets_query" { + source = "git::https://github.com/pagopa/azurerm.git//key_vault_secrets_query?ref=v1.0.58" + + resource_group = local.key_vault_resource_group + key_vault_name = local.key_vault_name + + secrets = [ + "appinsights-instrumentation-key" + ] +} diff --git a/src/k8s/serviceaccounts.tf b/src/k8s/serviceaccounts.tf new file mode 100644 index 000000000..a77390bf1 --- /dev/null +++ b/src/k8s/serviceaccounts.tf @@ -0,0 +1,79 @@ +resource "kubernetes_service_account" "azure_devops" { + metadata { + name = "azure-devops" + namespace = "kube-system" + } + automount_service_account_token = false +} + +resource "kubernetes_cluster_role" "cluster_deployer" { + metadata { + name = "cluster-deployer" + } + + rule { + api_groups = [""] + resources = ["services"] + verbs = ["get", "list", "watch", "create", "update", "patch", "delete"] + } + + rule { + api_groups = ["extensions", "apps"] + resources = ["deployments"] + verbs = ["get", "list", "watch", "create", "update", "patch", "delete"] + } +} + +resource "kubernetes_role_binding" "deployer_binding" { + depends_on = [ + kubernetes_namespace.selc + ] + + for_each = toset(var.rbac_namespaces) + + metadata { + name = "deployer-binding" + namespace = each.key + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cluster-deployer" + } + subject { + kind = "ServiceAccount" + name = "azure-devops" + namespace = "kube-system" + } +} + +data "kubernetes_secret" "azure_devops_secret" { + metadata { + name = kubernetes_service_account.azure_devops.default_secret_name + namespace = "kube-system" + } + binary_data = { + "ca.crt" = "" + "token" = "" + } +} + +#tfsec:ignore:AZU023 +resource "azurerm_key_vault_secret" "azure_devops_sa_token" { + depends_on = [kubernetes_service_account.azure_devops] + name = "aks-azure-devops-sa-token" + value = data.kubernetes_secret.azure_devops_secret.binary_data["token"] # base64 value + content_type = "text/plain" + + key_vault_id = local.key_vault_id +} + +#tfsec:ignore:AZU023 +resource "azurerm_key_vault_secret" "azure_devops_sa_cacrt" { + depends_on = [kubernetes_service_account.azure_devops] + name = "aks-azure-devops-sa-cacrt" + value = data.kubernetes_secret.azure_devops_secret.binary_data["ca.crt"] # base64 value + content_type = "text/plain" + + key_vault_id = local.key_vault_id +} diff --git a/src/k8s/subscriptions/DEV-SelfCare/backend.ini b/src/k8s/subscriptions/DEV-SelfCare/backend.ini new file mode 100644 index 000000000..fef2dbad8 --- /dev/null +++ b/src/k8s/subscriptions/DEV-SelfCare/backend.ini @@ -0,0 +1,2 @@ +resource_group_name="io-infra-rg" +storage_account_name="selcdstinfraterraform" diff --git a/src/k8s/subscriptions/DEV-SelfCare/terraform.tfvars b/src/k8s/subscriptions/DEV-SelfCare/terraform.tfvars new file mode 100644 index 000000000..be5fbfc3c --- /dev/null +++ b/src/k8s/subscriptions/DEV-SelfCare/terraform.tfvars @@ -0,0 +1,6 @@ +env = "dev" +env_short = "d" + +# ingress +ingress_replica_count = "2" +ingress_load_balancer_ip = "10.1.0.250" diff --git a/src/k8s/subscriptions/PROD-SelfCare/backend.ini b/src/k8s/subscriptions/PROD-SelfCare/backend.ini new file mode 100644 index 000000000..fb99d65a2 --- /dev/null +++ b/src/k8s/subscriptions/PROD-SelfCare/backend.ini @@ -0,0 +1,2 @@ +resource_group_name="io-infra-rg" +storage_account_name="selcpstinfraterraform" diff --git a/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars b/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars new file mode 100644 index 000000000..2d1e2f88e --- /dev/null +++ b/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars @@ -0,0 +1,6 @@ +env = "prod" +env_short = "p" + +# ingress +ingress_replica_count = "6" +ingress_load_balancer_ip = "10.1.0.250" diff --git a/src/k8s/subscriptions/UAT-SelfCare/backend.ini b/src/k8s/subscriptions/UAT-SelfCare/backend.ini new file mode 100644 index 000000000..2a9e78317 --- /dev/null +++ b/src/k8s/subscriptions/UAT-SelfCare/backend.ini @@ -0,0 +1,2 @@ +resource_group_name="io-infra-rg" +storage_account_name="selcustinfraterraform" diff --git a/src/k8s/subscriptions/UAT-SelfCare/terraform.tfvars b/src/k8s/subscriptions/UAT-SelfCare/terraform.tfvars new file mode 100644 index 000000000..41f0092b9 --- /dev/null +++ b/src/k8s/subscriptions/UAT-SelfCare/terraform.tfvars @@ -0,0 +1,6 @@ +env = "uat" +env_short = "u" + +# ingress +ingress_replica_count = "2" +ingress_load_balancer_ip = "10.1.0.250" diff --git a/src/k8s/terraform.sh b/src/k8s/terraform.sh new file mode 100644 index 000000000..632ec79c1 --- /dev/null +++ b/src/k8s/terraform.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# +# Apply the configuration relative to a given subscription +# Subscription are defined in ./subscription +# Usage: +# ./terraform.sh apply|destroy|plan ENV-SelfCare +# +# ./terraform.sh apply DEV-SelfCare +# ./terraform.sh apply UAT-SelfCare +# ./terraform.sh apply PROD-SelfCare + +BASHDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +WORKDIR="$BASHDIR" + +set -e + +COMMAND=$1 +SUBSCRIPTION=$2 +shift 2 +other=$@ + +if [ -z "${SUBSCRIPTION}" ]; then + printf "\e[1;31mYou must provide a subscription as first argument.\n" + exit 1 +fi + +if [ ! -d "${WORKDIR}/subscriptions/${SUBSCRIPTION}" ]; then + printf "\e[1;31mYou must provide a subscription for which a variable file is defined. You provided: '%s'.\n" "${SUBSCRIPTION}" > /dev/stderr + exit 1 +fi + +az account set -s "${SUBSCRIPTION}" + +# shellcheck disable=SC1090 +source "${WORKDIR}/subscriptions/${SUBSCRIPTION}/backend.ini" +source "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" + +# shellcheck disable=SC2154 +printf "Subscription: %s\n" "${SUBSCRIPTION}" +printf "Resource Group Name: %s\n" "${resource_group_name}" +printf "Storage Account Name: %s\n" "${storage_account_name}" + +export TF_VAR_k8s_apiserver_port="443" +export TF_VAR_k8s_apiserver_host="${aks_private_fqdn}" +export TF_VAR_k8s_kube_config_path="${kube_config_path}" + +# init terraform backend +terraform init -reconfigure \ + -backend-config="storage_account_name=${storage_account_name}" \ + -backend-config="resource_group_name=${resource_group_name}" + +export HELM_DEBUG=1 +if echo "plan apply refresh import output destroy" | grep -w ${COMMAND} > /dev/null; then + if [ ${COMMAND} = "output" ]; then + terraform ${COMMAND} $other + else + terraform ${COMMAND} --var-file="${WORKDIR}/subscriptions/${SUBSCRIPTION}/terraform.tfvars" $other + fi +else + echo "Action not allowed." + exit 1 +fi diff --git a/src/k8s/variables.tf b/src/k8s/variables.tf new file mode 100644 index 000000000..5ab27df8b --- /dev/null +++ b/src/k8s/variables.tf @@ -0,0 +1,56 @@ +variable "location" { + type = string + default = "westeurope" +} + +variable "prefix" { + type = string + default = "selc" +} + +variable "env" { + type = string +} + +variable "env_short" { + type = string +} + +variable "k8s_kube_config_path" { + type = string + default = "~/.kube/config" +} + +variable "k8s_apiserver_host" { + type = string +} + +variable "k8s_apiserver_port" { + type = number + default = 443 +} + +variable "k8s_apiserver_insecure" { + type = bool + default = false +} + +variable "rbac_namespaces" { + type = list(string) + default = ["selc"] +} + +# ingress + +variable "ingress_replica_count" { + type = string +} + +variable "ingress_load_balancer_ip" { + type = string +} + +variable "default_service_port" { + type = number + default = 8080 +} From 35196555a408fc9da8381a31e62d266e15d26167 Mon Sep 17 00:00:00 2001 From: anttorre Date: Fri, 15 Oct 2021 11:48:07 +0200 Subject: [PATCH 12/17] script file modified in order to be executable using cygwin --- src/k8s/scripts/setup.sh | 13 ++++++++++++- src/k8s/terraform.sh | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/k8s/scripts/setup.sh b/src/k8s/scripts/setup.sh index 9c83a53c3..b7785c8c3 100644 --- a/src/k8s/scripts/setup.sh +++ b/src/k8s/scripts/setup.sh @@ -33,6 +33,17 @@ aks_name=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:name}") aks_resource_group_name=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:resourceGroup}") aks_private_fqdn=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:privateFqdn}") +# in widows, even if using cygwin, these variables will contain a landing \r character +aks_name=${aks_name//[$'\r']} +aks_resource_group_name=${aks_resource_group_name//[$'\r']} +aks_private_fqdn=${aks_private_fqdn//[$'\r']} + +# if using cygwin, we have to transcode the WORKDIR +HOME_DIR=$HOME +if [[ $HOME_DIR == /cygdrive/* ]]; then + HOME_DIR=$(cygpath -w ~) +fi + rm -rf "${HOME}/.kube/config-${aks_name}" az aks get-credentials -g "${aks_resource_group_name}" -n "${aks_name}" --subscription "${SUBSCRIPTION}" --file "~/.kube/config-${aks_name}" az aks get-credentials -g "${aks_resource_group_name}" -n "${aks_name}" --subscription "${SUBSCRIPTION}" --overwrite-existing @@ -41,6 +52,6 @@ echo "kube_config_path=~/.kube/config-${aks_name}" >> "${WORKDIR}/subscriptions/ # with AAD auth enabled we need to authenticate the machine on the first setup echo "Follow Microsoft sign in steps. kubectl get pods command will fail but it's the expected behavior" -kubectl --kubeconfig="${HOME}/.kube/config-${aks_name}" get pods +kubectl --kubeconfig="${HOME_DIR}/.kube/config-${aks_name}" get pods kubectl config use-context "${aks_name}" kubectl get pods diff --git a/src/k8s/terraform.sh b/src/k8s/terraform.sh index 632ec79c1..873460a19 100644 --- a/src/k8s/terraform.sh +++ b/src/k8s/terraform.sh @@ -50,6 +50,12 @@ terraform init -reconfigure \ -backend-config="storage_account_name=${storage_account_name}" \ -backend-config="resource_group_name=${resource_group_name}" +# if using cygwin, we have to transcode the WORKDIR +if [[ $WORKDIR == /cygdrive/* ]]; then + WORKDIR=$(cygpath -w $WORKDIR) +fi + + export HELM_DEBUG=1 if echo "plan apply refresh import output destroy" | grep -w ${COMMAND} > /dev/null; then if [ ${COMMAND} = "output" ]; then From 86730c80738bb68f111de5493ad5d70fdc97989a Mon Sep 17 00:00:00 2001 From: anttorre Date: Mon, 18 Oct 2021 15:53:24 +0200 Subject: [PATCH 13/17] ingress PROD setted to 2 --- src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars b/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars index 2d1e2f88e..3baed6dab 100644 --- a/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars +++ b/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars @@ -2,5 +2,5 @@ env = "prod" env_short = "p" # ingress -ingress_replica_count = "6" +ingress_replica_count = "2" ingress_load_balancer_ip = "10.1.0.250" From c06f0885e6275ffb379263273c2a0fe830a63c19 Mon Sep 17 00:00:00 2001 From: anttorre Date: Mon, 18 Oct 2021 21:39:17 +0200 Subject: [PATCH 14/17] fixed bastianhost generation in win environment --- src/k8s/scripts/setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/k8s/scripts/setup.sh b/src/k8s/scripts/setup.sh index b7785c8c3..ff73c78cd 100644 --- a/src/k8s/scripts/setup.sh +++ b/src/k8s/scripts/setup.sh @@ -42,13 +42,14 @@ aks_private_fqdn=${aks_private_fqdn//[$'\r']} HOME_DIR=$HOME if [[ $HOME_DIR == /cygdrive/* ]]; then HOME_DIR=$(cygpath -w ~) + HOME_DIR=${HOME_DIR//\\//} fi rm -rf "${HOME}/.kube/config-${aks_name}" az aks get-credentials -g "${aks_resource_group_name}" -n "${aks_name}" --subscription "${SUBSCRIPTION}" --file "~/.kube/config-${aks_name}" az aks get-credentials -g "${aks_resource_group_name}" -n "${aks_name}" --subscription "${SUBSCRIPTION}" --overwrite-existing echo "aks_private_fqdn=${aks_private_fqdn}" >> "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" -echo "kube_config_path=~/.kube/config-${aks_name}" >> "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" +echo "kube_config_path=${HOME_DIR}/.kube/config-${aks_name}" >> "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" # with AAD auth enabled we need to authenticate the machine on the first setup echo "Follow Microsoft sign in steps. kubectl get pods command will fail but it's the expected behavior" From c792fdd24833999c35495b4226b2bead14d65937 Mon Sep 17 00:00:00 2001 From: anttorre Date: Tue, 19 Oct 2021 00:12:22 +0200 Subject: [PATCH 15/17] temporarely using local module to fix aks outbound permission --- src/core/aks.tf | 3 +- src/modules/kubernetes_cluster/main.tf | 147 ++++++++++++++ src/modules/kubernetes_cluster/outputs.tf | 27 +++ src/modules/kubernetes_cluster/variables.tf | 210 ++++++++++++++++++++ 4 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 src/modules/kubernetes_cluster/main.tf create mode 100644 src/modules/kubernetes_cluster/outputs.tf create mode 100644 src/modules/kubernetes_cluster/variables.tf diff --git a/src/core/aks.tf b/src/core/aks.tf index e5aee8be2..e4db406f8 100644 --- a/src/core/aks.tf +++ b/src/core/aks.tf @@ -5,7 +5,8 @@ resource "azurerm_resource_group" "rg_aks" { } module "aks" { - source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.60" + source = "../modules/kubernetes_cluster" +// source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.60" name = format("%s-aks", local.project) location = azurerm_resource_group.rg_aks.location diff --git a/src/modules/kubernetes_cluster/main.tf b/src/modules/kubernetes_cluster/main.tf new file mode 100644 index 000000000..0bfd2d3e6 --- /dev/null +++ b/src/modules/kubernetes_cluster/main.tf @@ -0,0 +1,147 @@ +resource "azurerm_kubernetes_cluster" "this" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + dns_prefix = var.dns_prefix + kubernetes_version = var.kubernetes_version + sku_tier = var.sku_tier + + private_cluster_enabled = var.private_cluster_enabled + + default_node_pool { + name = var.node_pool_name + vm_size = var.vm_size + availability_zones = var.availability_zones + vnet_subnet_id = var.vnet_subnet_id + enable_auto_scaling = var.enable_auto_scaling + node_count = var.node_count + min_count = var.min_count + max_count = var.max_count + max_pods = var.max_pods + + upgrade_settings { + max_surge = var.upgrade_settings_max_surge + } + + tags = var.tags + } + + automatic_channel_upgrade = var.automatic_channel_upgrade + api_server_authorized_ip_ranges = var.api_server_authorized_ip_ranges #tfsec:ignore:AZU008 + + identity { + type = "SystemAssigned" + } + + dynamic "network_profile" { + for_each = var.network_profile != null ? [var.network_profile] : [] + iterator = p + content { + docker_bridge_cidr = p.value.docker_bridge_cidr + dns_service_ip = p.value.dns_service_ip + network_policy = p.value.network_policy + network_plugin = p.value.network_plugin + outbound_type = p.value.outbound_type + service_cidr = p.value.service_cidr + load_balancer_sku = "Standard" + load_balancer_profile { + outbound_ip_address_ids = var.outbound_ip_address_ids + + } + } + } + + + role_based_access_control { + enabled = var.rbac_enabled + azure_active_directory { + managed = true + admin_group_object_ids = var.aad_admin_group_ids + } + } + + + addon_profile { + + oms_agent { + enabled = var.log_analytics_workspace_id != null ? true : false #tfsec:ignore:AZU009 + log_analytics_workspace_id = var.log_analytics_workspace_id + } + + aci_connector_linux { + enabled = false + } + + azure_policy { + enabled = false + } + + http_application_routing { + enabled = false + } + + kube_dashboard { + enabled = false + } + } + + tags = var.tags +} + +resource "azurerm_role_assignment" "aks" { + scope = azurerm_kubernetes_cluster.this.id + role_definition_name = "Monitoring Metrics Publisher" + principal_id = azurerm_kubernetes_cluster.this.addon_profile[0].oms_agent[0].oms_agent_identity[0].object_id +} + +resource "azurerm_role_assignment" "vnet_role" { + scope = var.vnet_id + role_definition_name = "Network Contributor" + principal_id = azurerm_kubernetes_cluster.this.identity[0].principal_id +} + +resource "azurerm_role_assignment" "vnet_outbound_role" { + for_each = toset(var.outbound_ip_address_ids) + + scope = each.key + role_definition_name = "Network Contributor" + principal_id = azurerm_kubernetes_cluster.this.identity[0].principal_id +} + +resource "azurerm_monitor_metric_alert" "this" { + for_each = var.metric_alerts + + name = format("%s-%s", azurerm_kubernetes_cluster.this.name, upper(each.key)) + resource_group_name = var.resource_group_name + scopes = [azurerm_kubernetes_cluster.this.id] + frequency = each.value.frequency + window_size = each.value.window_size + enabled = var.alerts_enabled + + dynamic "action" { + for_each = var.action + content { + # action_group_id - (required) is a type of string + action_group_id = action.value["action_group_id"] + # webhook_properties - (optional) is a type of map of string + webhook_properties = action.value["webhook_properties"] + } + } + + criteria { + aggregation = each.value.aggregation + metric_namespace = each.value.metric_namespace + metric_name = each.value.metric_name + operator = each.value.operator + threshold = each.value.threshold + + dynamic "dimension" { + for_each = each.value.dimension + content { + name = dimension.value.name + operator = dimension.value.operator + values = dimension.value.values + } + } + } +} \ No newline at end of file diff --git a/src/modules/kubernetes_cluster/outputs.tf b/src/modules/kubernetes_cluster/outputs.tf new file mode 100644 index 000000000..f3165c844 --- /dev/null +++ b/src/modules/kubernetes_cluster/outputs.tf @@ -0,0 +1,27 @@ +output "id" { + value = azurerm_kubernetes_cluster.this.id +} + +output "name" { + value = azurerm_kubernetes_cluster.this.name +} + +output "fqdn" { + value = azurerm_kubernetes_cluster.this.fqdn +} + +output "private_fqdn" { + value = azurerm_kubernetes_cluster.this.private_fqdn +} + +output "client_certificate" { + value = azurerm_kubernetes_cluster.this.kube_config.0.client_certificate +} + +output "kube_config" { + value = azurerm_kubernetes_cluster.this.kube_config_raw +} + +output "kubelet_identity_id" { + value = azurerm_kubernetes_cluster.this.kubelet_identity.0.object_id +} \ No newline at end of file diff --git a/src/modules/kubernetes_cluster/variables.tf b/src/modules/kubernetes_cluster/variables.tf new file mode 100644 index 000000000..9ce3e9408 --- /dev/null +++ b/src/modules/kubernetes_cluster/variables.tf @@ -0,0 +1,210 @@ +variable "name" { + type = string + description = "Cluster name" +} + +variable "dns_prefix" { + type = string + description = "Dns prefix." +} + +variable "resource_group_name" { + type = string + description = "Resource group name." +} + +variable "location" { + type = string +} + +variable "aad_admin_group_ids" { + description = "IDs of the Azure AD group for cluster-admin access" + type = list(string) +} + +variable "node_pool_name" { + type = string + default = "default" + description = "The name which should be used for the default Kubernetes Node Pool." +} + +variable "vm_size" { + type = string + default = "Standard_DS2_v2" + description = "The size of the Virtual Machine" +} + +variable "sku_tier" { + type = string + description = "The SKU Tier that should be used for this Kubernetes Cluster." + default = "Free" +} + +variable "kubernetes_version" { + type = string + description = "Version of Kubernetes specified when creating the AKS managed cluster." + default = null +} + +variable "availability_zones" { + type = list(string) + description = "A list of Availability Zones across which the Node Pool should be spread." + default = [] +} + +variable "private_cluster_enabled" { + type = bool + default = false + description = "Provides a Private IP Address for the Kubernetes API on the Virtual Network where the Kubernetes Cluster is located." +} + +variable "vnet_id" { + type = string + description = "Virtual network id, where the k8s cluster is deployed." +} + +variable "vnet_subnet_id" { + type = string + description = "The ID of a Subnet where the Kubernetes Node Pool should exist." + default = null +} + +variable "dns_prefix_private_cluster" { + type = string + description = "Specifies the DNS prefix to use with private clusters. Changing this forces a new resource to be created." + default = null +} + +variable "automatic_channel_upgrade" { + type = string + description = "The upgrade channel for this Kubernetes Cluster" + default = null +} + +variable "api_server_authorized_ip_ranges" { + type = list(string) + description = "The IP ranges to whitelist for incoming traffic to the masters." + default = [] +} + +variable "network_profile" { + type = object({ + docker_bridge_cidr = string + dns_service_ip = string + network_policy = string + network_plugin = string + outbound_type = string + service_cidr = string + }) + default = null +} + +variable "outbound_ip_address_ids" { + type = list(string) + default = [] + description = "The ID of the Public IP Addresses which should be used for outbound communication for the cluster load balancer." +} + +# Autoscaling + +variable "node_count" { + type = number + description = "The initial number of nodes which should exist in this Node Pool." + default = 1 +} + +variable "enable_auto_scaling" { + type = bool + description = "Should the Kubernetes Auto Scaler be enabled for this Node Pool? " + default = false +} + +variable "min_count" { + type = number + description = "The minimum number of nodes which should exist in this Node Pool. If specified this must be between 1 and 1000" + default = null +} + +variable "max_count" { + type = number + description = "The maximum number of nodes which should exist in this Node Pool. If specified this must be between 1 and 1000" + default = null +} + +variable "max_pods" { + type = number + description = "The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." + default = 30 +} + +variable "upgrade_settings_max_surge" { + type = string + description = "The maximum number or percentage of nodes which will be added to the Node Pool size during an upgrade." + default = "33%" +} + +# Logs +variable "log_analytics_workspace_id" { + type = string + description = "The ID of the Log Analytics Workspace which the OMS Agent should send data to." + default = null +} + +# Kubernetes RBAC +variable "rbac_enabled" { + type = bool + description = "Is Role Based Access Control Enabled?" + default = true +} + +variable "metric_alerts" { + default = {} + + description = < Date: Tue, 19 Oct 2021 15:00:46 +0200 Subject: [PATCH 16/17] k8s pagopa module upgrade --- src/core/aks.tf | 3 +- src/core/dns_private.tf | 2 +- src/modules/kubernetes_cluster/main.tf | 147 -------------- src/modules/kubernetes_cluster/outputs.tf | 27 --- src/modules/kubernetes_cluster/variables.tf | 210 -------------------- 5 files changed, 2 insertions(+), 387 deletions(-) delete mode 100644 src/modules/kubernetes_cluster/main.tf delete mode 100644 src/modules/kubernetes_cluster/outputs.tf delete mode 100644 src/modules/kubernetes_cluster/variables.tf diff --git a/src/core/aks.tf b/src/core/aks.tf index e4db406f8..cc834ecca 100644 --- a/src/core/aks.tf +++ b/src/core/aks.tf @@ -5,8 +5,7 @@ resource "azurerm_resource_group" "rg_aks" { } module "aks" { - source = "../modules/kubernetes_cluster" -// source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.60" + source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.75" name = format("%s-aks", local.project) location = azurerm_resource_group.rg_aks.location diff --git a/src/core/dns_private.tf b/src/core/dns_private.tf index 811058824..0d72cbf36 100644 --- a/src/core/dns_private.tf +++ b/src/core/dns_private.tf @@ -30,4 +30,4 @@ resource "azurerm_private_dns_zone_virtual_network_link" "privatelink_mongo_cosm registration_enabled = false tags = var.tags -} \ No newline at end of file +} diff --git a/src/modules/kubernetes_cluster/main.tf b/src/modules/kubernetes_cluster/main.tf deleted file mode 100644 index 0bfd2d3e6..000000000 --- a/src/modules/kubernetes_cluster/main.tf +++ /dev/null @@ -1,147 +0,0 @@ -resource "azurerm_kubernetes_cluster" "this" { - name = var.name - location = var.location - resource_group_name = var.resource_group_name - dns_prefix = var.dns_prefix - kubernetes_version = var.kubernetes_version - sku_tier = var.sku_tier - - private_cluster_enabled = var.private_cluster_enabled - - default_node_pool { - name = var.node_pool_name - vm_size = var.vm_size - availability_zones = var.availability_zones - vnet_subnet_id = var.vnet_subnet_id - enable_auto_scaling = var.enable_auto_scaling - node_count = var.node_count - min_count = var.min_count - max_count = var.max_count - max_pods = var.max_pods - - upgrade_settings { - max_surge = var.upgrade_settings_max_surge - } - - tags = var.tags - } - - automatic_channel_upgrade = var.automatic_channel_upgrade - api_server_authorized_ip_ranges = var.api_server_authorized_ip_ranges #tfsec:ignore:AZU008 - - identity { - type = "SystemAssigned" - } - - dynamic "network_profile" { - for_each = var.network_profile != null ? [var.network_profile] : [] - iterator = p - content { - docker_bridge_cidr = p.value.docker_bridge_cidr - dns_service_ip = p.value.dns_service_ip - network_policy = p.value.network_policy - network_plugin = p.value.network_plugin - outbound_type = p.value.outbound_type - service_cidr = p.value.service_cidr - load_balancer_sku = "Standard" - load_balancer_profile { - outbound_ip_address_ids = var.outbound_ip_address_ids - - } - } - } - - - role_based_access_control { - enabled = var.rbac_enabled - azure_active_directory { - managed = true - admin_group_object_ids = var.aad_admin_group_ids - } - } - - - addon_profile { - - oms_agent { - enabled = var.log_analytics_workspace_id != null ? true : false #tfsec:ignore:AZU009 - log_analytics_workspace_id = var.log_analytics_workspace_id - } - - aci_connector_linux { - enabled = false - } - - azure_policy { - enabled = false - } - - http_application_routing { - enabled = false - } - - kube_dashboard { - enabled = false - } - } - - tags = var.tags -} - -resource "azurerm_role_assignment" "aks" { - scope = azurerm_kubernetes_cluster.this.id - role_definition_name = "Monitoring Metrics Publisher" - principal_id = azurerm_kubernetes_cluster.this.addon_profile[0].oms_agent[0].oms_agent_identity[0].object_id -} - -resource "azurerm_role_assignment" "vnet_role" { - scope = var.vnet_id - role_definition_name = "Network Contributor" - principal_id = azurerm_kubernetes_cluster.this.identity[0].principal_id -} - -resource "azurerm_role_assignment" "vnet_outbound_role" { - for_each = toset(var.outbound_ip_address_ids) - - scope = each.key - role_definition_name = "Network Contributor" - principal_id = azurerm_kubernetes_cluster.this.identity[0].principal_id -} - -resource "azurerm_monitor_metric_alert" "this" { - for_each = var.metric_alerts - - name = format("%s-%s", azurerm_kubernetes_cluster.this.name, upper(each.key)) - resource_group_name = var.resource_group_name - scopes = [azurerm_kubernetes_cluster.this.id] - frequency = each.value.frequency - window_size = each.value.window_size - enabled = var.alerts_enabled - - dynamic "action" { - for_each = var.action - content { - # action_group_id - (required) is a type of string - action_group_id = action.value["action_group_id"] - # webhook_properties - (optional) is a type of map of string - webhook_properties = action.value["webhook_properties"] - } - } - - criteria { - aggregation = each.value.aggregation - metric_namespace = each.value.metric_namespace - metric_name = each.value.metric_name - operator = each.value.operator - threshold = each.value.threshold - - dynamic "dimension" { - for_each = each.value.dimension - content { - name = dimension.value.name - operator = dimension.value.operator - values = dimension.value.values - } - } - } -} \ No newline at end of file diff --git a/src/modules/kubernetes_cluster/outputs.tf b/src/modules/kubernetes_cluster/outputs.tf deleted file mode 100644 index f3165c844..000000000 --- a/src/modules/kubernetes_cluster/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -output "id" { - value = azurerm_kubernetes_cluster.this.id -} - -output "name" { - value = azurerm_kubernetes_cluster.this.name -} - -output "fqdn" { - value = azurerm_kubernetes_cluster.this.fqdn -} - -output "private_fqdn" { - value = azurerm_kubernetes_cluster.this.private_fqdn -} - -output "client_certificate" { - value = azurerm_kubernetes_cluster.this.kube_config.0.client_certificate -} - -output "kube_config" { - value = azurerm_kubernetes_cluster.this.kube_config_raw -} - -output "kubelet_identity_id" { - value = azurerm_kubernetes_cluster.this.kubelet_identity.0.object_id -} \ No newline at end of file diff --git a/src/modules/kubernetes_cluster/variables.tf b/src/modules/kubernetes_cluster/variables.tf deleted file mode 100644 index 9ce3e9408..000000000 --- a/src/modules/kubernetes_cluster/variables.tf +++ /dev/null @@ -1,210 +0,0 @@ -variable "name" { - type = string - description = "Cluster name" -} - -variable "dns_prefix" { - type = string - description = "Dns prefix." -} - -variable "resource_group_name" { - type = string - description = "Resource group name." -} - -variable "location" { - type = string -} - -variable "aad_admin_group_ids" { - description = "IDs of the Azure AD group for cluster-admin access" - type = list(string) -} - -variable "node_pool_name" { - type = string - default = "default" - description = "The name which should be used for the default Kubernetes Node Pool." -} - -variable "vm_size" { - type = string - default = "Standard_DS2_v2" - description = "The size of the Virtual Machine" -} - -variable "sku_tier" { - type = string - description = "The SKU Tier that should be used for this Kubernetes Cluster." - default = "Free" -} - -variable "kubernetes_version" { - type = string - description = "Version of Kubernetes specified when creating the AKS managed cluster." - default = null -} - -variable "availability_zones" { - type = list(string) - description = "A list of Availability Zones across which the Node Pool should be spread." - default = [] -} - -variable "private_cluster_enabled" { - type = bool - default = false - description = "Provides a Private IP Address for the Kubernetes API on the Virtual Network where the Kubernetes Cluster is located." -} - -variable "vnet_id" { - type = string - description = "Virtual network id, where the k8s cluster is deployed." -} - -variable "vnet_subnet_id" { - type = string - description = "The ID of a Subnet where the Kubernetes Node Pool should exist." - default = null -} - -variable "dns_prefix_private_cluster" { - type = string - description = "Specifies the DNS prefix to use with private clusters. Changing this forces a new resource to be created." - default = null -} - -variable "automatic_channel_upgrade" { - type = string - description = "The upgrade channel for this Kubernetes Cluster" - default = null -} - -variable "api_server_authorized_ip_ranges" { - type = list(string) - description = "The IP ranges to whitelist for incoming traffic to the masters." - default = [] -} - -variable "network_profile" { - type = object({ - docker_bridge_cidr = string - dns_service_ip = string - network_policy = string - network_plugin = string - outbound_type = string - service_cidr = string - }) - default = null -} - -variable "outbound_ip_address_ids" { - type = list(string) - default = [] - description = "The ID of the Public IP Addresses which should be used for outbound communication for the cluster load balancer." -} - -# Autoscaling - -variable "node_count" { - type = number - description = "The initial number of nodes which should exist in this Node Pool." - default = 1 -} - -variable "enable_auto_scaling" { - type = bool - description = "Should the Kubernetes Auto Scaler be enabled for this Node Pool? " - default = false -} - -variable "min_count" { - type = number - description = "The minimum number of nodes which should exist in this Node Pool. If specified this must be between 1 and 1000" - default = null -} - -variable "max_count" { - type = number - description = "The maximum number of nodes which should exist in this Node Pool. If specified this must be between 1 and 1000" - default = null -} - -variable "max_pods" { - type = number - description = "The maximum number of pods that can run on each agent. Changing this forces a new resource to be created." - default = 30 -} - -variable "upgrade_settings_max_surge" { - type = string - description = "The maximum number or percentage of nodes which will be added to the Node Pool size during an upgrade." - default = "33%" -} - -# Logs -variable "log_analytics_workspace_id" { - type = string - description = "The ID of the Log Analytics Workspace which the OMS Agent should send data to." - default = null -} - -# Kubernetes RBAC -variable "rbac_enabled" { - type = bool - description = "Is Role Based Access Control Enabled?" - default = true -} - -variable "metric_alerts" { - default = {} - - description = < Date: Tue, 19 Oct 2021 15:16:12 +0200 Subject: [PATCH 17/17] external groups are considered as admin in DEV environment --- src/core/aks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/aks.tf b/src/core/aks.tf index cc834ecca..b09b01990 100644 --- a/src/core/aks.tf +++ b/src/core/aks.tf @@ -23,7 +23,7 @@ module "aks" { private_cluster_enabled = true rbac_enabled = true - aad_admin_group_ids = var.env_short == "d" ? [data.azuread_group.adgroup_admin.object_id, data.azuread_group.adgroup_developers.object_id] : [data.azuread_group.adgroup_admin.object_id] + aad_admin_group_ids = var.env_short == "d" ? [data.azuread_group.adgroup_admin.object_id, data.azuread_group.adgroup_developers.object_id, data.azuread_group.adgroup_externals.object_id] : [data.azuread_group.adgroup_admin.object_id] vnet_id = module.vnet.id vnet_subnet_id = module.k8s_snet.id