diff --git a/.gitignore b/.gitignore index 5aadc243c..67d29c2d3 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ __TMP .metals/ __azurite_* /.idea +.bastianhost.ini diff --git a/src/core/aks.tf b/src/core/aks.tf index e5aee8be2..b09b01990 100644 --- a/src/core/aks.tf +++ b/src/core/aks.tf @@ -5,7 +5,7 @@ resource "azurerm_resource_group" "rg_aks" { } module "aks" { - source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.60" + source = "git::https://github.com/pagopa/azurerm.git//kubernetes_cluster?ref=v1.0.75" name = format("%s-aks", local.project) location = azurerm_resource_group.rg_aks.location @@ -23,7 +23,7 @@ module "aks" { private_cluster_enabled = true rbac_enabled = true - aad_admin_group_ids = var.env_short == "d" ? [data.azuread_group.adgroup_admin.object_id, data.azuread_group.adgroup_developers.object_id] : [data.azuread_group.adgroup_admin.object_id] + aad_admin_group_ids = var.env_short == "d" ? [data.azuread_group.adgroup_admin.object_id, data.azuread_group.adgroup_developers.object_id, data.azuread_group.adgroup_externals.object_id] : [data.azuread_group.adgroup_admin.object_id] vnet_id = module.vnet.id vnet_subnet_id = module.k8s_snet.id diff --git a/src/core/dns_private.tf b/src/core/dns_private.tf index 811058824..0d72cbf36 100644 --- a/src/core/dns_private.tf +++ b/src/core/dns_private.tf @@ -30,4 +30,4 @@ resource "azurerm_private_dns_zone_virtual_network_link" "privatelink_mongo_cosm registration_enabled = false tags = var.tags -} \ No newline at end of file +} diff --git a/src/k8s/.terraform.lock.hcl b/src/k8s/.terraform.lock.hcl new file mode 100644 index 000000000..3fdc33c40 --- /dev/null +++ b/src/k8s/.terraform.lock.hcl @@ -0,0 +1,78 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.5.0" + constraints = "2.5.0" + hashes = [ + "h1:Er35+K+GSrfZEJId/OqCWvOUa0idXQYyTrA2+I9KfI4=", + "zh:08e0ae5f1fde389a3cb9b32d3910fd0fe7cb6d361cf1133a22e803b7a7e66b8f", + "zh:093e70b0b4245605b6798be089defe385ac20e3a7f8aea64a7095bd4f762c5e9", + "zh:1fab548430864022308cb16b2fd9eebc993e63c1572aedd3ec9f81a2ebdc9e38", + "zh:5cc657d824b21f430a2c37d52c8a9a3ab06fdb3039a10eccd427bd5a6917ace1", + "zh:7863ab17f8cb12154d356d513e375772017904e6a7626ebb8e39210730afff6a", + "zh:7e53f8baa5a9279e4e7ed8533955f0e06bcbc8477fcb6a9bb22c10d5fc5c4d11", + "zh:91ed1dce045b6714cd8d1931e50347d1ef8ce5fc614229acc28527b0407a344b", + "zh:afce857b4eeb53f932ab0324e2f9ffb5014ece10e45dbfc5b7f09b9769123d90", + "zh:b8a81586ed314bdcfed05c5480a4135d4e251be2c41e8ba8026c669d951cb459", + "zh:db06bdf8d2d825399809e85a6b8be291e34416cda95e116e135f2ae605f7d2a0", + "zh:f5e03cf80fe30e0b1604ec0330ff2f2937acd512ae1a126684d6b61da2acb126", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "2.79.1" + constraints = "2.79.1" + hashes = [ + "h1:jPlACSoNUhzN7FpvgK8pze5IqtVt7Xw/0+kdzQmoIDI=", + "zh:120da7d501adb34c5600a09847e483e953cf45badb76c2e213f11beba5856bee", + "zh:2e1fb026cc2eb76aba2129661a9bdf0e4014668c2ea045208148b505aa708f6f", + "zh:3541d4daac9c07c6ffbb2f266c731efa32e08c3b72bd9b5454d368f418f3dd1c", + "zh:3efe2ee3cb51820d18ed5f3456f3ef8dcdfeb90d79790a1a1f8f3a8c2430e8a0", + "zh:46c1cc27031138f41e0bb4bee16df10431c963f23d0716583b7e66cf9be1b58a", + "zh:61125f3af098955320409dcb801a239059e3062937c9eded80cd2296bccabc3c", + "zh:681fc0457852db69dd1ce142c830e849f7d4293f7c6ab295bd493a1f7fb68133", + "zh:aa1e101f425f89a672e9821d11518fb93450d117ce6588852e87af369bdefc66", + "zh:be8120f98fcacaad9ae88986ea8018d715b03c00ff3fc23954af9c8be0c4e4f1", + "zh:d6c957b6fb43810a48c39ee907bdef2306b577a50df8cc231a6cd71650fb7009", + "zh:df243c69f4823935f34c2eb0a46f83050e2d78026bf953eacc580710dc4c1e40", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.2.0" + constraints = "~> 2.2.0" + hashes = [ + "h1:lFm6HwNEXgXT50K1jE7wnNaBLHTAt04KE5tjWQcJOMg=", + "zh:01341dd1e9cc7e7f6999e11e7473bcdca2dd72dd27f91beed1f4fb599a15dfba", + "zh:20e86c9eccd3a81ef5ac243af31b61fc4d2d679437384bd0870e92fa1b3ed6c9", + "zh:22a71127c5dbea4f62edb5bcf00b5c163de04aa19d45a7a1f621f973ffd09d20", + "zh:28ab7c84a5f8ed82fc520668db93d650571ddf59d98845cb18a1fa1a7888efc0", + "zh:3985a30929ad8fdc6b94f0e1cbd62a63db75ee961b8ba7db1cf4bfd29e8009ff", + "zh:477d92e26ba0c906087a5dd827ac3917dad7d5af770ee0ab4b08d0f273150586", + "zh:750928ec5ef54b2090bd6a6d8a19630a8712bbbccc0429251e88ccd361c1d3c0", + "zh:a615841fd90094bddc1269127e501fa60453c441b9548ff73752fe14efc38ed0", + "zh:e762aca7883374fa255efba50f5bdf791fece7d61e3920e593fb1a2cbb598981", + "zh:f76f372ead52948ca53610b371cb80c80ebcf058ef0a5c0ce9f0ce38dcc9a8eb", + "zh:fa36fe93ed977f4478cc6547ec3c45c28e56f10632e85446b0c3d71449f8c4bb", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.3.2" + constraints = "~> 2.3.2" + hashes = [ + "h1:D8HWX3vouTPI3Jicq43xOQyoYWtSsVua92cBVrJ3ZMs=", + "zh:10f71c170be13538374a4b9553fcb3d98a6036bcd1ca5901877773116c3f828e", + "zh:11d2230e531b7480317e988207a73cb67b332f225b0892304983b19b6014ebe0", + "zh:3317387a9a6cc27fd7536b8f3cad4b8a9285e9461f125c5a15d192cef3281856", + "zh:458a9858362900fbe97e00432ae8a5bef212a4dacf97a57ede7534c164730da4", + "zh:50ea297007d9fe53e5411577f87a4b13f3877ce732089b42f938430e6aadff0d", + "zh:56705c959e4cbea3b115782d04c62c68ac75128c5c44ee7aa4043df253ffbfe3", + "zh:7eb3722f7f036e224824470c3e0d941f1f268fcd5fa2f8203e0eee425d0e1484", + "zh:9f408a6df4d74089e6ce18f9206b06b8107ddb57e2bc9b958a6b7dc352c62980", + "zh:aadd25ccc3021040808feb2645779962f638766eb583f586806e59f24dde81bb", + "zh:b101c3456e4309b09aab129b0118561178c92cb4be5d96dec553189c3084dca1", + "zh:ec08478573b4953764099fbfd670fae81dc24b60e467fb3b023e6fab50b70a9e", + ] +} diff --git a/src/k8s/README.md b/src/k8s/README.md new file mode 100644 index 000000000..7c8b41a41 --- /dev/null +++ b/src/k8s/README.md @@ -0,0 +1,204 @@ +# kubernetes-infrastructure + +This is a kubernetes infrastructure configuration. + +## Requirements + +### 1. terraform + +In order to manage the suitable version of terraform it is strongly recommended to install the following tool: + +- [tfenv](https://github.com/tfutils/tfenv): **Terraform** version manager inspired by rbenv. + +Once these tools have been installed, install the terraform version shown in: + +- .terraform-version + +After installation install terraform: + +```sh +tfenv install +``` + +### 2. Azure CLI + +In order to authenticate to Azure portal and manage terraform state it's necessary to install and login to Azure subscription. + +- [Azure CLI](https://docs.microsoft.com/it-it/cli/azure/install-azure-cli) + +After installation login to Azure: + +```sh +az login +``` + +### 3. kubectl + +In order to run commands against Kubernetes clusters it's necessary to install kubectl. + +- [kubectl](https://kubernetes.io/docs/tasks/tools/) + +### 4. helm + +In order to use Helm package manager for Kubernetes it's necessary to install helm. + +- [helm](https://helm.sh/docs/helm/helm_install/) + +### 5. Access to bastian host (jumpbox) + +We deploy a kubernetes in private mode so it is not public accessible. +We use an SSH connection to a bastian host started on demand (jumpbox). + +```sh +## ~/.ssh/config file configuration +# Change project_aks_env_user, user and bastian_host_env_ip with correct values +# Ask to an Azure Administrator the id_rsa_project_aks_env_user private key +Host project_aks_env_user + AddKeysToAgent yes + UseKeychain yes + HostName bastian_host_env_ip + User user + IdentityFile ~/.ssh/id_rsa_project_aks_env_user +``` + +```sh +# set rw permission to id_rsa_project_aks_env_user key only for current user +chmod 600 ~/.ssh/id_rsa_project_aks_env_user +ssh-add ~/.ssh/id_rsa_project_aks_env_user +# if nedded, restart ssh-agent +eval "$(ssh-agent -s)" +``` + +## Terraform modules + +As PagoPA we build our standard Terraform modules, check available modules: + +- [PagoPA Terraform modules](https://github.com/search?q=topic%3Aterraform-modules+org%3Apagopa&type=repositories) + +## Setup configuration + +Before first use we need to run a setup script to configure `.bastianhost.ini` and download kube config. + +```sh +bash scripts/setup.sh ENV-PROJECT + +# example for SelfCare project in DEV environment +bash scripts/setup.sh DEV-SelfCare +``` + +## Apply changes + +To apply changes use `terraform.sh` script as follow: + +```sh +bash terraform.sh apply|plan|destroy ENV-PROJECT + +# example to apply configuration for SelfCare project in DEV environment +bash terraform.sh apply DEV-SelfCare +``` + +## Terraform lock.hcl + +We have both developers who work with your Terraform configuration on their Linux, macOS or Windows workstations and automated systems that apply the configuration while running on Linux. +https://www.terraform.io/docs/cli/commands/providers/lock.html#specifying-target-platforms + +So we need to specify this in terraform lock providers: + +```sh +terraform init + +rm .terraform.lock.hcl + +terraform providers lock \ + -platform=windows_amd64 \ + -platform=darwin_amd64 \ + -platform=linux_amd64 +``` + +## Precommit checks + +Check your code before commit. + +https://github.com/antonbabenko/pre-commit-terraform#how-to-install + +```sh +pre-commit run -a +``` + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >=0.15.3 | +| [azuread](#requirement\_azuread) | = 1.6.0 | +| [azurerm](#requirement\_azurerm) | ~> 2.60.0 | +| [helm](#requirement\_helm) | ~> 2.1.2 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.3.2 | + +## Providers + +| Name | Version | +|------|---------| +| [azuread](#provider\_azuread) | 1.6.0 | +| [azurerm](#provider\_azurerm) | 2.60.0 | +| [helm](#provider\_helm) | 2.1.2 | +| [kubernetes](#provider\_kubernetes) | 2.3.2 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [key\_vault\_secrets\_query](#module\_key\_vault\_secrets\_query) | git::https://github.com/pagopa/azurerm.git//key_vault_secrets_query | v1.0.58 | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_key_vault_secret.azure_devops_sa_cacrt](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.azure_devops_sa_token](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [helm_release.ingress](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_cluster_role.cluster_deployer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | +| [kubernetes_cluster_role.view_extra](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | +| [kubernetes_cluster_role_binding.edit_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | +| [kubernetes_cluster_role_binding.view_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | +| [kubernetes_cluster_role_binding.view_extra_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | +| [kubernetes_namespace.selc](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_namespace.ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_role_binding.deployer_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource | +| [kubernetes_secret.azure-storage](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | +| [kubernetes_secret.selc-application-insights](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | +| [kubernetes_service_account.azure_devops](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource | +| [azuread_group.adgroup_contributors](https://registry.terraform.io/providers/hashicorp/azuread/1.6.0/docs/data-sources/group) | data source | +| [azuread_group.adgroup_externals](https://registry.terraform.io/providers/hashicorp/azuread/1.6.0/docs/data-sources/group) | data source | +| [azuread_group.adgroup_security](https://registry.terraform.io/providers/hashicorp/azuread/1.6.0/docs/data-sources/group) | data source | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | +| [kubernetes_secret.azure_devops_secret](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/data-sources/secret) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [env](#input\_env) | n/a | `string` | n/a | yes | +| [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | +| [ingress\_load\_balancer\_ip](#input\_ingress\_load\_balancer\_ip) | n/a | `string` | n/a | yes | +| [ingress\_replica\_count](#input\_ingress\_replica\_count) | n/a | `string` | n/a | yes | +| [k8s\_apiserver\_host](#input\_k8s\_apiserver\_host) | n/a | `string` | n/a | yes | +| [rbac\_namespaces](#input\_rbac\_namespaces) | n/a | `list(string)` | n/a | yes | +| [default\_service\_port](#input\_default\_service\_port) | n/a | `number` | `8080` | no | +| [event\_hub\_port](#input\_event\_hub\_port) | n/a | `number` | `9093` | no | +| [k8s\_apiserver\_insecure](#input\_k8s\_apiserver\_insecure) | n/a | `bool` | `false` | no | +| [k8s\_apiserver\_port](#input\_k8s\_apiserver\_port) | n/a | `number` | `443` | no | +| [k8s\_kube\_config\_path](#input\_k8s\_kube\_config\_path) | n/a | `string` | `"~/.kube/config"` | no | +| [location](#input\_location) | n/a | `string` | `"westeurope"` | no | +| [prefix](#input\_prefix) | n/a | `string` | `"selc"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [azure\_devops\_sa\_cacrt](#output\_azure\_devops\_sa\_cacrt) | n/a | +| [azure\_devops\_sa\_token](#output\_azure\_devops\_sa\_token) | n/a | + diff --git a/src/k8s/ingress.tf b/src/k8s/ingress.tf new file mode 100644 index 000000000..ff39b68c3 --- /dev/null +++ b/src/k8s/ingress.tf @@ -0,0 +1,32 @@ +# from Microsoft docs https://docs.microsoft.com/it-it/azure/aks/ingress-internal-ip +resource "helm_release" "ingress" { + name = "nginx-ingress" + repository = "https://kubernetes.github.io/ingress-nginx" + chart = "ingress-nginx" + version = "3.31.0" + namespace = kubernetes_namespace.ingress.metadata[0].name + + values = [ + "${templatefile("${path.module}/ingress/loadbalancer.yaml.tpl", { load_balancer_ip = var.ingress_load_balancer_ip })}" + ] + + set { + name = "controller.replicaCount" + value = var.ingress_replica_count + } + + set { + name = "controller.nodeSelector.beta\\.kubernetes\\.io/os" + value = "linux" + } + + set { + name = "defaultBackend.nodeSelector.beta\\.kubernetes\\.io/os" + value = "linux" + } + + set { + name = "controller.admissionWebhooks.patch.nodeSelector.beta\\.kubernetes\\.io/os" + value = "linux" + } +} diff --git a/src/k8s/ingress/loadbalancer.yaml.tpl b/src/k8s/ingress/loadbalancer.yaml.tpl new file mode 100644 index 000000000..f00cb77ca --- /dev/null +++ b/src/k8s/ingress/loadbalancer.yaml.tpl @@ -0,0 +1,5 @@ +controller: + service: + loadBalancerIP: ${load_balancer_ip} + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" diff --git a/src/k8s/locals.tf b/src/k8s/locals.tf new file mode 100644 index 000000000..73e58dfa2 --- /dev/null +++ b/src/k8s/locals.tf @@ -0,0 +1,6 @@ +locals { + project = format("%s-%s", var.prefix, var.env_short) + key_vault_name = format("%s-kv", local.project) + key_vault_resource_group = format("%s-sec-rg", local.project) + key_vault_id = "${data.azurerm_subscription.current.id}/resourceGroups/${local.key_vault_resource_group}/providers/Microsoft.KeyVault/vaults/${local.key_vault_name}" +} diff --git a/src/k8s/main.tf b/src/k8s/main.tf new file mode 100644 index 000000000..48a55cc6e --- /dev/null +++ b/src/k8s/main.tf @@ -0,0 +1,48 @@ +terraform { + required_version = ">=0.15.3" + + backend "azurerm" { + container_name = "k8sstate" + key = "terraform.tfstate" + } + + required_providers { + azurerm = { + version = "= 2.79.1" + } + azuread = { + source = "hashicorp/azuread" + version = "= 2.5.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.3.2" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.2.0" + } + } +} + +provider "kubernetes" { + host = "https://${var.k8s_apiserver_host}:${var.k8s_apiserver_port}" + insecure = var.k8s_apiserver_insecure + config_path = var.k8s_kube_config_path +} + +provider "helm" { + kubernetes { + host = "https://${var.k8s_apiserver_host}:${var.k8s_apiserver_port}" + insecure = var.k8s_apiserver_insecure + config_path = var.k8s_kube_config_path + } +} + +provider "azurerm" { + features {} +} + +data "azurerm_subscription" "current" {} + +data "azurerm_client_config" "current" {} diff --git a/src/k8s/namespaces.tf b/src/k8s/namespaces.tf new file mode 100644 index 000000000..446a61c3a --- /dev/null +++ b/src/k8s/namespaces.tf @@ -0,0 +1,11 @@ +resource "kubernetes_namespace" "ingress" { + metadata { + name = "ingress" + } +} + +resource "kubernetes_namespace" "selc" { + metadata { + name = "selc" + } +} diff --git a/src/k8s/outputs.tf b/src/k8s/outputs.tf new file mode 100644 index 000000000..35d297504 --- /dev/null +++ b/src/k8s/outputs.tf @@ -0,0 +1,9 @@ +output "azure_devops_sa_token" { + value = data.kubernetes_secret.azure_devops_secret.binary_data["token"] + sensitive = true +} + +output "azure_devops_sa_cacrt" { + value = data.kubernetes_secret.azure_devops_secret.binary_data["ca.crt"] + sensitive = true +} diff --git a/src/k8s/rbac.tf b/src/k8s/rbac.tf new file mode 100644 index 000000000..840cae54d --- /dev/null +++ b/src/k8s/rbac.tf @@ -0,0 +1,176 @@ +data "azuread_group" "adgroup_externals" { + display_name = format("%s-adgroup-externals", local.project) +} + +data "azuread_group" "adgroup_developers" { + display_name = format("%s-adgroup-developers", local.project) +} + +data "azuread_group" "adgroup_security" { + display_name = format("%s-adgroup-security", local.project) +} + +data "azuread_group" "adgroup_operations" { + display_name = format("%s-adgroup-operations", local.project) +} + +data "azuread_group" "adgroup_technical_project_managers" { + display_name = format("%s-adgroup-technical-project-managers", local.project) +} + +resource "kubernetes_cluster_role" "view_extra" { + metadata { + name = "view-extra" + } + + dynamic "rule" { + for_each = var.env_short == "d" ? [""] : [] + + content { + api_groups = [""] + resources = ["pods/attach", "pods/exec", "pods/portforward", "pods/proxy", "secrets", "services/proxy"] + verbs = ["get", "list", "watch"] + } + } + + dynamic "rule" { + for_each = var.env_short == "d" ? [""] : [] + content { + api_groups = [""] + resources = ["pods/attach", "pods/exec", "pods/portforward", "pods/proxy"] + verbs = ["create", "delete", "deletecollection", "patch", "update"] + } + } +} + +resource "kubernetes_cluster_role_binding" "view_extra_binding" { + metadata { + name = "view-extra-binding" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.view_extra.metadata[0].name + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_security.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_developers.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_externals.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_operations.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_technical_project_managers.object_id + namespace = "kube-system" + } +} + +resource "kubernetes_cluster_role" "edit_extra" { + metadata { + name = "edit-extra" + } + + rule { + api_groups = ["rbac.authorization.k8s.io"] + resources = ["*"] + verbs = ["get", "list"] + } +} + +resource "kubernetes_cluster_role_binding" "edit_extra_binding" { + metadata { + name = "edit-extra-binding" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.edit_extra.metadata[0].name + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_developers.object_id + namespace = "kube-system" + } +} + +resource "kubernetes_cluster_role_binding" "edit_binding" { + metadata { + name = "edit-binding" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "edit" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_developers.object_id + namespace = "kube-system" + } +} + +resource "kubernetes_cluster_role_binding" "view_binding" { + metadata { + name = "view-binding" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "view" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_developers.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_security.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_externals.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_operations.object_id + namespace = "kube-system" + } + + subject { + kind = "Group" + name = data.azuread_group.adgroup_technical_project_managers.object_id + namespace = "kube-system" + } +} diff --git a/src/k8s/scripts/base_64_decode.sh b/src/k8s/scripts/base_64_decode.sh new file mode 100644 index 000000000..e6d7996ce --- /dev/null +++ b/src/k8s/scripts/base_64_decode.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ $# -lt 1 ]; then + echo 1>&2 "$0: missed string to decode." + exit 2 +fi + +str=$1 + +echo Encoding $str +echo "" +echo "$str" | base64 --decode diff --git a/src/k8s/scripts/restart-pods.sh b/src/k8s/scripts/restart-pods.sh new file mode 100644 index 000000000..e8e731b94 --- /dev/null +++ b/src/k8s/scripts/restart-pods.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ $# -lt 1 ]; then + echo 1>&2 "$0: missed namespace!! usage: $0 " + exit 2 +elif [ $# -gt 1 ]; then + echo 1>&2 "$0: too many arguments" + exit 2 +fi + +namespace=$1 +waitfor=3s + +deploys=`kubectl -n $namespace get deployments | tail -n +2 | cut -d ' ' -f 1` +for deploy in $deploys; do + kubectl -n $1 rollout restart deployments/$deploy + sleep $waitfor +done diff --git a/src/k8s/scripts/setup.sh b/src/k8s/scripts/setup.sh new file mode 100644 index 000000000..ff73c78cd --- /dev/null +++ b/src/k8s/scripts/setup.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# +# Setup configuration relative to a given subscription +# Subscription are defined in ./subscription +# Usage: +# ./setup.sh ENV-SelfCare +# +# ./setup.sh DEV-SelfCare +# ./setup.sh UAT-SelfCare +# ./setup.sh PROD-SelfCare + +BASHDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +WORKDIR="${BASHDIR//scripts/}" + +set -e + +SUBSCRIPTION=$1 + +if [ -z "${SUBSCRIPTION}" ]; then + printf "\e[1;31mYou must provide a subscription as first argument.\n" + exit 1 +fi + +if [ ! -d "${WORKDIR}/subscriptions/${SUBSCRIPTION}" ]; then + printf "\e[1;31mYou must provide a subscription for which a variable file is defined. You provided: '%s'.\n" "${SUBSCRIPTION}" > /dev/stderr + exit 1 +fi + +az account set -s "${SUBSCRIPTION}" + +aks_name=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:name}") +aks_resource_group_name=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:resourceGroup}") +aks_private_fqdn=$(az aks list -o tsv --query "[?contains(name,'aks')].{Name:privateFqdn}") + +# in widows, even if using cygwin, these variables will contain a landing \r character +aks_name=${aks_name//[$'\r']} +aks_resource_group_name=${aks_resource_group_name//[$'\r']} +aks_private_fqdn=${aks_private_fqdn//[$'\r']} + +# if using cygwin, we have to transcode the WORKDIR +HOME_DIR=$HOME +if [[ $HOME_DIR == /cygdrive/* ]]; then + HOME_DIR=$(cygpath -w ~) + HOME_DIR=${HOME_DIR//\\//} +fi + +rm -rf "${HOME}/.kube/config-${aks_name}" +az aks get-credentials -g "${aks_resource_group_name}" -n "${aks_name}" --subscription "${SUBSCRIPTION}" --file "~/.kube/config-${aks_name}" +az aks get-credentials -g "${aks_resource_group_name}" -n "${aks_name}" --subscription "${SUBSCRIPTION}" --overwrite-existing +echo "aks_private_fqdn=${aks_private_fqdn}" >> "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" +echo "kube_config_path=${HOME_DIR}/.kube/config-${aks_name}" >> "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" + +# with AAD auth enabled we need to authenticate the machine on the first setup +echo "Follow Microsoft sign in steps. kubectl get pods command will fail but it's the expected behavior" +kubectl --kubeconfig="${HOME_DIR}/.kube/config-${aks_name}" get pods +kubectl config use-context "${aks_name}" +kubectl get pods diff --git a/src/k8s/scripts/ssh-port-forward.sh b/src/k8s/scripts/ssh-port-forward.sh new file mode 100644 index 000000000..6aa47ed4d --- /dev/null +++ b/src/k8s/scripts/ssh-port-forward.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# From https://dev.to/jaysonsantos/using-terraform-s-remote-exec-provider-with-aws-ssm-5po + +set -ex +test -n "$DESTINATION_IP" || (echo missing DESTINATION_IP; exit 1) +test -n "$USERNAME" || (echo missing USERNAME; exit 1) +test -n "$RANDOM_PORT" || (echo missing RANDOM_PORT; exit 1) +test -n "$TARGET" || (echo missing TARGET; exit 1) + +set +e + +cleanup() { + cat log.txt + rm -rf log.txt + exit $! +} + +for try in {0..5}; do + echo "Trying to port forward retry #$try" + # The following command MUST NOT print to the stdio otherwise it will just + # inherit the pipe from the parent process and will hold terraform's lock + ssh -f -o StrictHostKeyChecking=no \ + -o ControlMaster=no \ + "$USERNAME@$DESTINATION_IP" \ + -L "127.0.0.1:$RANDOM_PORT:$TARGET" \ + sleep 15m &> log.txt # This is the special ingredient! + success="$?" + if [ "$success" -eq 0 ]; then + cleanup 0 + fi + sleep 5s +done + +echo "Failed to start a port forwarding session" +cleanup 1 diff --git a/src/k8s/secrets.tf b/src/k8s/secrets.tf new file mode 100644 index 000000000..88561cf75 --- /dev/null +++ b/src/k8s/secrets.tf @@ -0,0 +1,10 @@ +module "key_vault_secrets_query" { + source = "git::https://github.com/pagopa/azurerm.git//key_vault_secrets_query?ref=v1.0.58" + + resource_group = local.key_vault_resource_group + key_vault_name = local.key_vault_name + + secrets = [ + "appinsights-instrumentation-key" + ] +} diff --git a/src/k8s/serviceaccounts.tf b/src/k8s/serviceaccounts.tf new file mode 100644 index 000000000..a77390bf1 --- /dev/null +++ b/src/k8s/serviceaccounts.tf @@ -0,0 +1,79 @@ +resource "kubernetes_service_account" "azure_devops" { + metadata { + name = "azure-devops" + namespace = "kube-system" + } + automount_service_account_token = false +} + +resource "kubernetes_cluster_role" "cluster_deployer" { + metadata { + name = "cluster-deployer" + } + + rule { + api_groups = [""] + resources = ["services"] + verbs = ["get", "list", "watch", "create", "update", "patch", "delete"] + } + + rule { + api_groups = ["extensions", "apps"] + resources = ["deployments"] + verbs = ["get", "list", "watch", "create", "update", "patch", "delete"] + } +} + +resource "kubernetes_role_binding" "deployer_binding" { + depends_on = [ + kubernetes_namespace.selc + ] + + for_each = toset(var.rbac_namespaces) + + metadata { + name = "deployer-binding" + namespace = each.key + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cluster-deployer" + } + subject { + kind = "ServiceAccount" + name = "azure-devops" + namespace = "kube-system" + } +} + +data "kubernetes_secret" "azure_devops_secret" { + metadata { + name = kubernetes_service_account.azure_devops.default_secret_name + namespace = "kube-system" + } + binary_data = { + "ca.crt" = "" + "token" = "" + } +} + +#tfsec:ignore:AZU023 +resource "azurerm_key_vault_secret" "azure_devops_sa_token" { + depends_on = [kubernetes_service_account.azure_devops] + name = "aks-azure-devops-sa-token" + value = data.kubernetes_secret.azure_devops_secret.binary_data["token"] # base64 value + content_type = "text/plain" + + key_vault_id = local.key_vault_id +} + +#tfsec:ignore:AZU023 +resource "azurerm_key_vault_secret" "azure_devops_sa_cacrt" { + depends_on = [kubernetes_service_account.azure_devops] + name = "aks-azure-devops-sa-cacrt" + value = data.kubernetes_secret.azure_devops_secret.binary_data["ca.crt"] # base64 value + content_type = "text/plain" + + key_vault_id = local.key_vault_id +} diff --git a/src/k8s/subscriptions/DEV-SelfCare/backend.ini b/src/k8s/subscriptions/DEV-SelfCare/backend.ini new file mode 100644 index 000000000..fef2dbad8 --- /dev/null +++ b/src/k8s/subscriptions/DEV-SelfCare/backend.ini @@ -0,0 +1,2 @@ +resource_group_name="io-infra-rg" +storage_account_name="selcdstinfraterraform" diff --git a/src/k8s/subscriptions/DEV-SelfCare/terraform.tfvars b/src/k8s/subscriptions/DEV-SelfCare/terraform.tfvars new file mode 100644 index 000000000..be5fbfc3c --- /dev/null +++ b/src/k8s/subscriptions/DEV-SelfCare/terraform.tfvars @@ -0,0 +1,6 @@ +env = "dev" +env_short = "d" + +# ingress +ingress_replica_count = "2" +ingress_load_balancer_ip = "10.1.0.250" diff --git a/src/k8s/subscriptions/PROD-SelfCare/backend.ini b/src/k8s/subscriptions/PROD-SelfCare/backend.ini new file mode 100644 index 000000000..fb99d65a2 --- /dev/null +++ b/src/k8s/subscriptions/PROD-SelfCare/backend.ini @@ -0,0 +1,2 @@ +resource_group_name="io-infra-rg" +storage_account_name="selcpstinfraterraform" diff --git a/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars b/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars new file mode 100644 index 000000000..3baed6dab --- /dev/null +++ b/src/k8s/subscriptions/PROD-SelfCare/terraform.tfvars @@ -0,0 +1,6 @@ +env = "prod" +env_short = "p" + +# ingress +ingress_replica_count = "2" +ingress_load_balancer_ip = "10.1.0.250" diff --git a/src/k8s/subscriptions/UAT-SelfCare/backend.ini b/src/k8s/subscriptions/UAT-SelfCare/backend.ini new file mode 100644 index 000000000..2a9e78317 --- /dev/null +++ b/src/k8s/subscriptions/UAT-SelfCare/backend.ini @@ -0,0 +1,2 @@ +resource_group_name="io-infra-rg" +storage_account_name="selcustinfraterraform" diff --git a/src/k8s/subscriptions/UAT-SelfCare/terraform.tfvars b/src/k8s/subscriptions/UAT-SelfCare/terraform.tfvars new file mode 100644 index 000000000..41f0092b9 --- /dev/null +++ b/src/k8s/subscriptions/UAT-SelfCare/terraform.tfvars @@ -0,0 +1,6 @@ +env = "uat" +env_short = "u" + +# ingress +ingress_replica_count = "2" +ingress_load_balancer_ip = "10.1.0.250" diff --git a/src/k8s/terraform.sh b/src/k8s/terraform.sh new file mode 100644 index 000000000..873460a19 --- /dev/null +++ b/src/k8s/terraform.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# +# Apply the configuration relative to a given subscription +# Subscription are defined in ./subscription +# Usage: +# ./terraform.sh apply|destroy|plan ENV-SelfCare +# +# ./terraform.sh apply DEV-SelfCare +# ./terraform.sh apply UAT-SelfCare +# ./terraform.sh apply PROD-SelfCare + +BASHDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +WORKDIR="$BASHDIR" + +set -e + +COMMAND=$1 +SUBSCRIPTION=$2 +shift 2 +other=$@ + +if [ -z "${SUBSCRIPTION}" ]; then + printf "\e[1;31mYou must provide a subscription as first argument.\n" + exit 1 +fi + +if [ ! -d "${WORKDIR}/subscriptions/${SUBSCRIPTION}" ]; then + printf "\e[1;31mYou must provide a subscription for which a variable file is defined. You provided: '%s'.\n" "${SUBSCRIPTION}" > /dev/stderr + exit 1 +fi + +az account set -s "${SUBSCRIPTION}" + +# shellcheck disable=SC1090 +source "${WORKDIR}/subscriptions/${SUBSCRIPTION}/backend.ini" +source "${WORKDIR}/subscriptions/${SUBSCRIPTION}/.bastianhost.ini" + +# shellcheck disable=SC2154 +printf "Subscription: %s\n" "${SUBSCRIPTION}" +printf "Resource Group Name: %s\n" "${resource_group_name}" +printf "Storage Account Name: %s\n" "${storage_account_name}" + +export TF_VAR_k8s_apiserver_port="443" +export TF_VAR_k8s_apiserver_host="${aks_private_fqdn}" +export TF_VAR_k8s_kube_config_path="${kube_config_path}" + +# init terraform backend +terraform init -reconfigure \ + -backend-config="storage_account_name=${storage_account_name}" \ + -backend-config="resource_group_name=${resource_group_name}" + +# if using cygwin, we have to transcode the WORKDIR +if [[ $WORKDIR == /cygdrive/* ]]; then + WORKDIR=$(cygpath -w $WORKDIR) +fi + + +export HELM_DEBUG=1 +if echo "plan apply refresh import output destroy" | grep -w ${COMMAND} > /dev/null; then + if [ ${COMMAND} = "output" ]; then + terraform ${COMMAND} $other + else + terraform ${COMMAND} --var-file="${WORKDIR}/subscriptions/${SUBSCRIPTION}/terraform.tfvars" $other + fi +else + echo "Action not allowed." + exit 1 +fi diff --git a/src/k8s/variables.tf b/src/k8s/variables.tf new file mode 100644 index 000000000..5ab27df8b --- /dev/null +++ b/src/k8s/variables.tf @@ -0,0 +1,56 @@ +variable "location" { + type = string + default = "westeurope" +} + +variable "prefix" { + type = string + default = "selc" +} + +variable "env" { + type = string +} + +variable "env_short" { + type = string +} + +variable "k8s_kube_config_path" { + type = string + default = "~/.kube/config" +} + +variable "k8s_apiserver_host" { + type = string +} + +variable "k8s_apiserver_port" { + type = number + default = 443 +} + +variable "k8s_apiserver_insecure" { + type = bool + default = false +} + +variable "rbac_namespaces" { + type = list(string) + default = ["selc"] +} + +# ingress + +variable "ingress_replica_count" { + type = string +} + +variable "ingress_load_balancer_ip" { + type = string +} + +variable "default_service_port" { + type = number + default = 8080 +}