Pull request aws#4: DELIVERY-8215 update to 0.27.0

Merge in DEL/karpenter-fork from DELIVERY-8215_update_to_0.27.0 to main

* commit 'c5b7235218996a4579180ab4811afe82c884ee9f':
  chore: Bump `karpenter-core` to latest (aws#3548)
  BREAKING CHANGE: Convert preferred anti-affinity to required anti-affinity (aws#3543)
  test: Fix daemonset preemption in testing to be `PreemptLowerPriority` (aws#3544)
  chore: Move instance to providers (aws#3509)
  docs: cleanup `aws.tags` and `specs.tags` docs (aws#3540)
  docs: Add Canva to Adopters (aws#3541)
  fix: fixed an issue where only using owners in amiSelector resulted in a failure (aws#3533)
  feat: add featureGates.driftEnabled to helm chart values (aws#3521)
  chore: Release v0.26.1 (aws#3519)
  fix: inflight IP tracking when CreateFleet returns an error (aws#3515)
  chore: add more integration test logs (aws#3511)
  chore(deps): bump github.com/aws/aws-sdk-go from 1.44.195 to 1.44.212 in /test (aws#3508)
  test: if the test fails, identify the names of the created nodes (aws#3504)
  chore: Move instancetypes to providers (aws#3478)
  feat: Added additional topology spread constraint (aws#3190)
  chore: Release v0.26.0 (aws#3506)
  test: fix test panic (aws#3503)
  chore: fix pco allocation strategy for ec2 fleet (aws#3502)
  docs: Add Docker to adopters list (aws#3500)
  Typo (aws#3487)

ADOPTERS.md

@@ -20,9 +20,11 @@ If you are open to others contacting you about your use of Karpenter on Slack, a
 | Anthropic | Better utilizing mixes of instance types for more reliable capacity | N/A | [Homepage](https://anthropic.com) |
 | Astradot | Using Karpenter on all K8s clusters in production and staging | N/A | [Homepage](https://astradot.com) |
 | Beeswax | Using Karpenter to scale our high load AdTech platform efficiently | `@James Wojewoda` | [Homepage](https://www.beeswax.com)
+| Canva | Using Karpetner to scale CPU and GPU workloads on EKS | `@groodt` | [Canva](https://www.canva.com/) |
 | Codefresh | Juggling workloads for the SAAS CD/GitOps offering | `@Vadim Gusev` | [Codefresh](https://codefresh.io/) |
 | Cordial   | Using Karpenter to scale multiple EKS clusters quickly | `@dschaaff` | [Cordial](https://cordial.com) |
 | Dig Security | Protecting our customers data - Using Karpenter to manage production and development workloads on EKS, We are using only Spot Instances in production. | `@Shahar Danus` | [Dig Security](https://dig.security/) |
+| Docker | Using Karpenter to scale Docker Hub on our EKS clusters | N/A | [Docker](https://www.docker.com) |
 | H2O.ai | Dynamically scaling CPU and GPU nodes for AI workloads  | `@Ophir Zahavi`, `@Asaf Oren` | [H2O.ai](https://h2o.ai/) |
 | idealo | Scaling multi-arch IPv6 clusters hosting web and event-driven applications | `@Heiko Rothe` | [Homepage](https://www.idealo.de) |
 | Nexxiot | Easier, Safer, Cleaner Global Transportation - Using Karpenter to manage EKS nodes | `@Alex Berger` | [Homepage](https://nexxiot.com/) |

charts/karpenter-crd/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: karpenter-crd
 description: A Helm chart for Karpenter Custom Resource Definitions (CRDs)
 type: application
-version: 0.25.0
-appVersion: 0.25.0
+version: 0.26.1
+appVersion: 0.26.1
 keywords:
   - cluster
   - node

charts/karpenter/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: karpenter
 description: A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes.
 type: application
-version: 0.25.0
-appVersion: 0.25.0
+version: 0.26.1
+appVersion: 0.26.1
 keywords:
   - cluster
   - node

charts/karpenter/README.md

@@ -2,20 +2,20 @@
 
 A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes.
 
-![Version: 0.25.0](https://img.shields.io/badge/Version-0.25.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.25.0](https://img.shields.io/badge/AppVersion-0.25.0-informational?style=flat-square)
+![Version: 0.26.1](https://img.shields.io/badge/Version-0.26.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.26.1](https://img.shields.io/badge/AppVersion-0.26.1-informational?style=flat-square)
 
 ## Documentation
 
-For full Karpenter documentation please checkout [https://karpenter.sh](https://karpenter.sh/v0.25.0/).
+For full Karpenter documentation please checkout [https://karpenter.sh](https://karpenter.sh/v0.26.1/).
 
 ## Installing the Chart
 
-You can follow the detailed installation instruction in the [documentation](https://karpenter.sh/v0.25.0/getting-started/getting-started-with-eksctl/#install) which covers the Karpenter prerequisites and installation options. The outcome of these instructions should result in something like the following command.
+You can follow the detailed installation instruction in the [documentation](https://karpenter.sh/v0.26.1/getting-started/getting-started-with-eksctl/#install) which covers the Karpenter prerequisites and installation options. The outcome of these instructions should result in something like the following command.
 
 ```bash
 helm upgrade --install --namespace karpenter --create-namespace \
   karpenter oci://public.ecr.aws/karpenter/karpenter \
-  --version v0.25.0 \
+  --version v0.26.1 \
   --set serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN} \
   --set settings.aws.clusterName=${CLUSTER_NAME} \
   --set settings.aws.clusterEndpoint=${CLUSTER_ENDPOINT} \
@@ -31,15 +31,15 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | additionalAnnotations | object | `{}` | Additional annotations to add into metadata. |
 | additionalClusterRoleRules | list | `[]` | Specifies additional rules for the core ClusterRole. |
 | additionalLabels | object | `{}` | Additional labels to add into metadata. |
-| affinity | object | `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"karpenter.sh/provisioner-name","operator":"DoesNotExist"}]}]}}}` | Affinity rules for scheduling the pod. |
+| affinity | object | `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"karpenter.sh/provisioner-name","operator":"DoesNotExist"}]}]}},"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"app.kubernetes.io/instance","operator":"In","values":["{{ .Release.Name }}"]}]},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity rules for scheduling the pod. |
 | controller.env | list | `[]` | Additional environment variables for the controller pod. |
 | controller.envFrom | list | `[]` |  |
 | controller.errorOutputPaths | list | `["stderr"]` | Controller errorOutputPaths - default to stderr only |
 | controller.extraVolumeMounts | list | `[]` | Additional volumeMounts for the controller pod. |
 | controller.healthProbe.port | int | `8081` | The container port to use for http health probe. |
-| controller.image.digest | string | `"sha256:fefae2739efa1c4d9561069d1683a0ed201e41771351da1ef504c805941f0bf2"` | SHA256 digest of the controller image. |
+| controller.image.digest | string | `"sha256:5dfe506624961f386b68556dd1cc850bfe3a42b62d2dd5dcb8b21d1a89ec817c"` | SHA256 digest of the controller image. |
 | controller.image.repository | string | `"public.ecr.aws/karpenter/controller"` | Repository path to the controller image. |
-| controller.image.tag | string | `"v0.25.0"` | Tag of the controller image. |
+| controller.image.tag | string | `"v0.26.1"` | Tag of the controller image. |
 | controller.logEncoding | string | `""` | Controller log encoding, defaults to the global log encoding |
 | controller.logLevel | string | `""` | Controller log level, defaults to the global log level |
 | controller.metrics.port | int | `8080` | The container port to use for metrics. |
@@ -50,7 +50,7 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | controller.sidecarVolumeMounts | list | `[]` | Additional volumeMounts for the sidecar - this will be added to the volume mounts on top of extraVolumeMounts |
 | dnsConfig | object | `{}` | Configure DNS Config for the pod |
 | dnsPolicy | string | `"Default"` | Configure the DNS Policy for the pod |
-| extraObjects | list | `[]` | Array of extra K8s manifests to deploy  |
+| extraObjects | list | `[]` | Array of extra K8s manifests to deploy |
 | extraVolumes | list | `[]` | Additional volumes for the pod. |
 | fullnameOverride | string | `""` | Overrides the chart's computed fullname. |
 | hostNetwork | bool | `false` | Bind the pod to the host network. This is required when using a custom CNI. |
@@ -74,7 +74,7 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | serviceMonitor.additionalLabels | object | `{}` | Additional labels for the ServiceMonitor. |
 | serviceMonitor.enabled | bool | `false` | Specifies whether a ServiceMonitor should be created. |
 | serviceMonitor.endpointConfig | object | `{}` | Endpoint configuration for the ServiceMonitor. |
-| settings | object | `{"aws":{"clusterEndpoint":"","clusterName":"","defaultInstanceProfile":"","enableENILimitedPodDensity":true,"enablePodENI":false,"interruptionQueueName":"","isolatedVPC":false,"nodeNameConvention":"ip-name","tags":null,"vmMemoryOverheadPercent":0.075},"batchIdleDuration":"1s","batchMaxDuration":"10s"}` | Global Settings to configure Karpenter |
+| settings | object | `{"aws":{"clusterEndpoint":"","clusterName":"","defaultInstanceProfile":"","enableENILimitedPodDensity":true,"enablePodENI":false,"interruptionQueueName":"","isolatedVPC":false,"nodeNameConvention":"ip-name","tags":null,"vmMemoryOverheadPercent":0.075},"batchIdleDuration":"1s","batchMaxDuration":"10s","featureGates":{"driftEnabled":false}}` | Global Settings to configure Karpenter |
 | settings.aws | object | `{"clusterEndpoint":"","clusterName":"","defaultInstanceProfile":"","enableENILimitedPodDensity":true,"enablePodENI":false,"interruptionQueueName":"","isolatedVPC":false,"nodeNameConvention":"ip-name","tags":null,"vmMemoryOverheadPercent":0.075}` | AWS-specific configuration values |
 | settings.aws.clusterEndpoint | string | `""` | Cluster endpoint. If not set, will be discovered during startup (EKS only) |
 | settings.aws.clusterName | string | `""` | Cluster name. |
@@ -88,6 +88,8 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | settings.aws.vmMemoryOverheadPercent | float | `0.075` | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types |
 | settings.batchIdleDuration | string | `"1s"` | The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. |
 | settings.batchMaxDuration | string | `"10s"` | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. |
+| settings.featureGates | object | `{"driftEnabled":false}` | Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features |
+| settings.featureGates.driftEnabled | bool | `false` | driftEnabled is in ALPHA and is disabled by default. Setting driftEnabled to true enables the drift deprovisioner to watch for drift between currently deployed nodes and the desired state of nodes set in provisioners and node templates |
 | strategy | object | `{"rollingUpdate":{"maxUnavailable":1}}` | Strategy for updating the pod. |
 | terminationGracePeriodSeconds | string | `nil` | Override the default termination grace period for the pod. |
 | tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"}]` | Tolerations to allow the pod to be scheduled to nodes with taints. |

charts/karpenter/templates/deployment.yaml

@@ -140,7 +140,7 @@ spec:
       {{- end }}
       {{- with .Values.affinity }}
       affinity:
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       {{- with .Values.tolerations }}
       tolerations:

charts/karpenter/values.yaml

@@ -72,6 +72,15 @@ affinity:
         - matchExpressions:
             - key: karpenter.sh/provisioner-name
               operator: DoesNotExist
+  podAntiAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchExpressions:
+            - key: app.kubernetes.io/instance
+              operator: In
+              values:
+                - "{{ .Release.Name }}"
+        topologyKey: "kubernetes.io/hostname"
 # -- topologySpreadConstraints to increase the controller resilience
 topologySpreadConstraints:
   - maxSkew: 1
@@ -92,7 +101,7 @@ extraVolumes: []
 #         expirationSeconds: 86400
 #         path: token
 
-# -- Array of extra K8s manifests to deploy 
+# -- Array of extra K8s manifests to deploy
 extraObjects: []
 #- apiVersion: karpenter.k8s.aws/v1alpha1
 #  kind: AWSNodeTemplate
@@ -109,17 +118,16 @@ controller:
     # -- Repository path to the controller image.
     repository: public.ecr.aws/karpenter/controller
     # -- Tag of the controller image.
-    tag: v0.25.0
+    tag: v0.26.1
     # -- SHA256 digest of the controller image.
-    digest: sha256:fefae2739efa1c4d9561069d1683a0ed201e41771351da1ef504c805941f0bf2
+    digest: sha256:5dfe506624961f386b68556dd1cc850bfe3a42b62d2dd5dcb8b21d1a89ec817c
   # -- SecurityContext for the controller container.
   securityContext: {}
   # -- Additional environment variables for the controller pod.
   env: []
   # - name: AWS_REGION
   #   value: eu-west-1
   envFrom: []
-
   # -- Resources for the controller pod.
   resources: {}
   # We usually recommend not to specify default resources and to leave this as a conscious
@@ -200,3 +208,10 @@ settings:
     interruptionQueueName: ""
     # -- The global tags to use on all AWS infrastructure resources (launch templates, instances, etc.) across node templates
     tags:
+  # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates
+  # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features
+  featureGates:
+    # -- driftEnabled is in ALPHA and is disabled by default.
+    # Setting driftEnabled to true enables the drift deprovisioner to watch for drift between currently deployed nodes
+    # and the desired state of nodes set in provisioners and node templates
+    driftEnabled: false

cmd/controller/main.go

@@ -41,7 +41,13 @@ func main() {
 		EventRecorder:       operator.EventRecorder,
 		StartAsync:          operator.Elected(),
 	})
-	awsCloudProvider := cloudprovider.New(awsCtx)
+	awsCloudProvider := cloudprovider.New(
+		awsCtx,
+		awsCtx.InstanceTypesProvider,
+		awsCtx.InstanceProvider,
+		awsCtx.KubeClient,
+		awsCtx.AMIProvider,
+	)
 	lo.Must0(operator.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe))
 	cloudProvider := metrics.Decorate(awsCloudProvider)
 

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.44.195
-	github.com/aws/karpenter-core v0.25.1-0.20230224225813-3e5ec9b8eb1a
+	github.com/aws/karpenter-core v0.27.0
 	github.com/go-playground/validator/v10 v10.11.2
 	github.com/imdario/mergo v0.3.13
 	github.com/mitchellh/hashstructure/v2 v2.0.2

go.sum

@@ -70,8 +70,8 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.44.195 h1:d5xFL0N83Fpsq2LFiHgtBUHknCRUPGHdOlCWt/jtOJs=
 github.com/aws/aws-sdk-go v1.44.195/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/karpenter-core v0.25.1-0.20230224225813-3e5ec9b8eb1a h1:+SrX+qETmZxfGFx0gWiHzUWsZytRbXuq3GGcH2z3AQk=
-github.com/aws/karpenter-core v0.25.1-0.20230224225813-3e5ec9b8eb1a/go.mod h1:szIXYuMzg8kAwAbMIoB1VJBjAM+AdX+cIwsINkmWnpI=
+github.com/aws/karpenter-core v0.27.0 h1:WaJ+7mIkyMH7+wT+g+XJalOc4aSY8+WFh7DoJvcomEc=
+github.com/aws/karpenter-core v0.27.0/go.mod h1:szIXYuMzg8kAwAbMIoB1VJBjAM+AdX+cIwsINkmWnpI=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

hack/api-code-gen.sh

@@ -8,7 +8,7 @@ fi
 echo "api-code-gen running ENABLE_GIT_PUSH: ${ENABLE_GIT_PUSH}"
 
 bandwidth() {
-  GENERATED_FILE="pkg/cloudprovider/zz_generated.bandwidth.go"
+  GENERATED_FILE="pkg/providers/instancetype/zz_generated.bandwidth.go"
   NO_UPDATE=''
   SUBJECT="Bandwidth"
 
@@ -30,7 +30,7 @@ pricing() {
 }
 
 vpcLimits() {
-  GENERATED_FILE="pkg/cloudprovider/zz_generated.vpclimits.go"
+  GENERATED_FILE="pkg/providers/instancetype/zz_generated.vpclimits.go"
   NO_UPDATE=''
   SUBJECT="VPC Limits"
 

hack/code/bandwidth_gen.go

@@ -40,7 +40,7 @@ var uriSelectors = map[string]string{
 
 const fileFormat = `
 %s
-package cloudprovider
+package instancetype
 
 // GENERATED FILE. DO NOT EDIT DIRECTLY.
 // Update hack/code/bandwidth_gen.go and re-generate to edit
@@ -56,7 +56,7 @@ var (
 func main() {
 	flag.Parse()
 	if flag.NArg() != 1 {
-		log.Fatalf("Usage: `bandwidth_gen.go pkg/cloudprovider/zz_generated.pricing.go`")
+		log.Fatalf("Usage: `bandwidth_gen.go pkg/providers/instancetype/zz_generated.pricing.go`")
 	}
 
 	bandwidth := map[string]int64{}

hack/code/vpc_limits_gen.go

@@ -35,7 +35,7 @@ func main() {
 	opts := options{}
 	flag.StringVar(&opts.urlInput, "url", "https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/master/pkg/aws/vpc/limits.go",
 		"url of the raw vpc/limits.go file in the github.com/aws/amazon-vpc-resource-controller-k8s repo")
-	flag.StringVar(&opts.sourceOutput, "output", "pkg/cloudprovider/zz_generated.vpclimits.go", "output location for the generated go source file")
+	flag.StringVar(&opts.sourceOutput, "output", "pkg/providers/instancetype/zz_generated.vpclimits.go", "output location for the generated go source file")
 	flag.Parse()
 
 	limitsURL, err := url.Parse(opts.urlInput)
@@ -58,7 +58,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	newRespData := strings.Replace(string(respData), "package vpc", "package cloudprovider", 1)
+	newRespData := strings.Replace(string(respData), "package vpc", "package instancetype", 1)
 	out.WriteString(newRespData)
 	defer out.Close()
 

hack/docs/instancetypes_gen_docs.go

@@ -233,9 +233,10 @@ func (f kubeDnsTransport) RoundTrip(request *http.Request) (*http.Response, erro
 }
 
 func NewAWSCloudProviderForCodeGen(ctx context.Context) *awscloudprovider.CloudProvider {
-	return awscloudprovider.New(awscontext.NewOrDie(cloudprovider.Context{
+	context := awscontext.NewOrDie(cloudprovider.Context{
 		Context:             ctx,
 		RESTConfig:          &rest.Config{},
 		KubernetesInterface: lo.Must(kubernetes.NewForConfigAndClient(&rest.Config{}, &http.Client{Transport: &kubeDnsTransport{}})),
-	}))
+	})
+	return awscloudprovider.New(context, context.InstanceTypesProvider, context.InstanceProvider, context.KubeClient, context.AMIProvider)
 }

hack/docs/metrics_gen_docs.go

@@ -81,7 +81,7 @@ description: >
 
 	// TODO @joinnis: Remove this line when exposing machine metrics
 	allMetrics = lo.Reject(allMetrics, func(m metricInfo, _ int) bool {
-		return m.subsystem == "machines"
+		return m.subsystem == "machines" || strings.HasPrefix(m.name, "controller_runtime")
 	})
 	for _, metric := range allMetrics {
 		if metric.subsystem != previousSubsystem {