From 2c333026668155a8f48f68e67b30d9098aa06465 Mon Sep 17 00:00:00 2001 From: Michael Barrientos Date: Tue, 31 Oct 2023 09:36:21 -0700 Subject: [PATCH 1/3] feat: Add Karpenter v1beta1 compatibility --- examples/karpenter/README.md | 2 ++ examples/karpenter/main.tf | 49 +++++++++++++++++++++++++++++++++++- modules/karpenter/main.tf | 6 +++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 6549a3e8d2..b4c6969cdc 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -81,6 +81,8 @@ Note that this example may create resources which cost money. Run `terraform des |------|------| | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_class](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index b925de7855..57a2f678e4 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -182,7 +182,7 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.29.0" + version = "v0.32.0" set { name = "settings.aws.clusterName" @@ -234,6 +234,31 @@ resource "kubectl_manifest" "karpenter_provisioner" { ] } +resource "kubectl_manifest" "karpenter_node_pool" { + yaml_body = <<-YAML + apiVersion: karpenter.sh/v1beta1 + kind: NodePool + metadata: + name: default + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + limits: + cpu: 1000 + nodeClassRef: + name: default + disruption: + consolidationPolicy: WhenEmpty + consolidationAfter: 30s + YAML + + depends_on = [ + helm_release.karpenter + ] +} + resource "kubectl_manifest" "karpenter_node_template" { yaml_body = <<-YAML apiVersion: karpenter.k8s.aws/v1alpha1 @@ -254,6 +279,28 @@ resource "kubectl_manifest" "karpenter_node_template" { ] } +resource "kubectl_manifest" "karpenter_node_class" { + yaml_body = <<-YAML + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + metadata: + name: default + spec: + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: ${module.eks.cluster_name} + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: ${module.eks.cluster_name} + tags: + karpenter.sh/discovery: ${module.eks.cluster_name} + YAML + + depends_on = [ + helm_release.karpenter + ] +} + # Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container) # and starts with zero replicas resource "kubectl_manifest" "karpenter_example_deployment" { diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 223c400b5d..6887c4784d 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -84,6 +84,12 @@ data "aws_iam_policy_document" "irsa" { "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeAvailabilityZones", "ec2:DescribeSpotPriceHistory", + "iam:AddRoleToInstanceProfile", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:GetInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:TagInstanceProfile", "pricing:GetProducts", ] From 8b6d4ae2fc5debb2cc17505a02e83f0f21096118 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 31 Oct 2023 17:51:41 -0400 Subject: [PATCH 2/3] fix: Update to make changes opt-in --- .pre-commit-config.yaml | 2 +- examples/karpenter/README.md | 4 +- examples/karpenter/main.tf | 98 ++++++++++------------------------ modules/karpenter/README.md | 1 + modules/karpenter/main.tf | 26 ++++++--- modules/karpenter/variables.tf | 6 +++ 6 files changed, 56 insertions(+), 81 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15d3d3efd7..145baf94e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.4 + rev: v1.83.5 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index b4c6969cdc..06b29e7bb4 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -73,7 +73,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | | [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | ## Resources @@ -83,8 +83,6 @@ Note that this example may create resources which cost money. Run `terraform des | [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_node_class](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 57a2f678e4..276b45dc4c 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -160,10 +160,9 @@ module "karpenter" { cluster_name = module.eks.cluster_name irsa_oidc_provider_arn = module.eks.oidc_provider_arn - # Used to attach additional IAM policies to the Karpenter controller IRSA role - # policies = { - # "xxx" = "yyy" - # } + # In v0.32.0/v1beta1, Karpenter now creates the IAM instance profile + # so we disable the Terraform creation and add the necessary permissions for Karpenter IRSA + enable_karpenter_instance_profile_creation = true # Used to attach additional IAM policies to the Karpenter node IAM role iam_role_additional_policies = { @@ -182,56 +181,27 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.32.0" - - set { - name = "settings.aws.clusterName" - value = module.eks.cluster_name - } + version = "v0.32.1" + + values = [ + <<-EOT + settings: + aws: + clusterName: ${module.eks.cluster_name} + clusterEndpoint: ${module.eks.cluster_endpoint} + interruptionQueueName: ${module.karpenter.queue_name} + tolerations: + - key: eks.amazonaws.com/compute-type + value: fargate + effect: NoSchedule + EOT + ] - set { - name = "settings.aws.clusterEndpoint" - value = module.eks.cluster_endpoint - } set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" value = module.karpenter.irsa_arn } - - set { - name = "settings.aws.defaultInstanceProfile" - value = module.karpenter.instance_profile_name - } - - set { - name = "settings.aws.interruptionQueueName" - value = module.karpenter.queue_name - } -} - -resource "kubectl_manifest" "karpenter_provisioner" { - yaml_body = <<-YAML - apiVersion: karpenter.sh/v1alpha5 - kind: Provisioner - metadata: - name: default - spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - resources: - cpu: 1000 - providerRef: - name: default - ttlSecondsAfterEmpty: 30 - YAML - - depends_on = [ - helm_release.karpenter - ] } resource "kubectl_manifest" "karpenter_node_pool" { @@ -251,27 +221,13 @@ resource "kubectl_manifest" "karpenter_node_pool" { name: default disruption: consolidationPolicy: WhenEmpty - consolidationAfter: 30s - YAML - - depends_on = [ - helm_release.karpenter - ] -} - -resource "kubectl_manifest" "karpenter_node_template" { - yaml_body = <<-YAML - apiVersion: karpenter.k8s.aws/v1alpha1 - kind: AWSNodeTemplate - metadata: - name: default - spec: - subnetSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - securityGroupSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} + consolidateAfter: 30s + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default YAML depends_on = [ @@ -286,6 +242,8 @@ resource "kubectl_manifest" "karpenter_node_class" { metadata: name: default spec: + amiFamily: AL2 + role: ${module.karpenter.role_arn} subnetSelectorTerms: - tags: karpenter.sh/discovery: ${module.eks.cluster_name} @@ -339,7 +297,7 @@ resource "kubectl_manifest" "karpenter_example_deployment" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 4.0" + version = "~> 5.0" name = local.name cidr = local.vpc_cidr diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 4a7f9c7f74..7b813471ad 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -150,6 +150,7 @@ No modules. | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `true` | no | | [create\_irsa](#input\_create\_irsa) | Determines whether an IAM role for service accounts is created | `bool` | `true` | no | +| [enable\_karpenter\_instance\_profile\_creation](#input\_enable\_karpenter\_instance\_profile\_creation) | Determines whether Karpenter will be allowed to create the IAM instance profile (v1beta1) or if Terraform will (v1alpha1) | `bool` | `false` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 6887c4784d..5cf4d39fd3 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -84,12 +84,6 @@ data "aws_iam_policy_document" "irsa" { "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeAvailabilityZones", "ec2:DescribeSpotPriceHistory", - "iam:AddRoleToInstanceProfile", - "iam:CreateInstanceProfile", - "iam:DeleteInstanceProfile", - "iam:GetInstanceProfile", - "iam:RemoveRoleFromInstanceProfile", - "iam:TagInstanceProfile", "pricing:GetProducts", ] @@ -166,6 +160,24 @@ data "aws_iam_policy_document" "irsa" { resources = [aws_sqs_queue.this[0].arn] } } + + # TODO - this will be replaced in v20.0 with the scoped policy provided by Karpenter + # https://github.com/aws/karpenter/blob/main/website/content/en/docs/upgrading/v1beta1-controller-policy.json + dynamic "statement" { + for_each = var.enable_karpenter_instance_profile_creation ? [1] : [] + + content { + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:GetInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:TagInstanceProfile", + ] + resources = ["*"] + } + } } resource "aws_iam_policy" "irsa" { @@ -374,7 +386,7 @@ locals { } resource "aws_iam_instance_profile" "this" { - count = var.create && var.create_instance_profile ? 1 : 0 + count = var.create && var.create_instance_profile && !var.enable_karpenter_instance_profile_creation ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index c025237b36..4a8389671b 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -123,6 +123,12 @@ variable "irsa_assume_role_condition_test" { default = "StringEquals" } +variable "enable_karpenter_instance_profile_creation" { + description = "Determines whether Karpenter will be allowed to create the IAM instance profile (v1beta1) or if Terraform will (v1alpha1)" + type = bool + default = false +} + ################################################################################ # Node Termination Queue ################################################################################ From 57d28b76ec73575c3cd2b088a633f8643f12e887 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 1 Nov 2023 11:27:40 -0400 Subject: [PATCH 3/3] fix: Update resource schemas to align with latest Karpenter version - validated and working as intended --- examples/karpenter/main.tf | 89 ++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 276b45dc4c..b8d7be97ff 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -186,53 +186,14 @@ resource "helm_release" "karpenter" { values = [ <<-EOT settings: - aws: - clusterName: ${module.eks.cluster_name} - clusterEndpoint: ${module.eks.cluster_endpoint} - interruptionQueueName: ${module.karpenter.queue_name} - tolerations: - - key: eks.amazonaws.com/compute-type - value: fargate - effect: NoSchedule + clusterName: ${module.eks.cluster_name} + clusterEndpoint: ${module.eks.cluster_endpoint} + interruptionQueueName: ${module.karpenter.queue_name} + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: ${module.karpenter.irsa_arn} EOT ] - - - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter.irsa_arn - } -} - -resource "kubectl_manifest" "karpenter_node_pool" { - yaml_body = <<-YAML - apiVersion: karpenter.sh/v1beta1 - kind: NodePool - metadata: - name: default - spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - cpu: 1000 - nodeClassRef: - name: default - disruption: - consolidationPolicy: WhenEmpty - consolidateAfter: 30s - template: - spec: - nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 - kind: EC2NodeClass - name: default - YAML - - depends_on = [ - helm_release.karpenter - ] } resource "kubectl_manifest" "karpenter_node_class" { @@ -243,7 +204,7 @@ resource "kubectl_manifest" "karpenter_node_class" { name: default spec: amiFamily: AL2 - role: ${module.karpenter.role_arn} + role: ${module.karpenter.role_name} subnetSelectorTerms: - tags: karpenter.sh/discovery: ${module.eks.cluster_name} @@ -259,6 +220,42 @@ resource "kubectl_manifest" "karpenter_node_class" { ] } +resource "kubectl_manifest" "karpenter_node_pool" { + yaml_body = <<-YAML + apiVersion: karpenter.sh/v1beta1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s + YAML + + depends_on = [ + kubectl_manifest.karpenter_node_class + ] +} + # Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container) # and starts with zero replicas resource "kubectl_manifest" "karpenter_example_deployment" {