Merge pull request #1187 from kinvolk/invidian/conntrack

Expose --conntrack-max-per-core kube-proxy flag

assets/charts/control-plane/kubernetes/templates/kube-proxy.yaml

@@ -42,6 +42,9 @@ spec:
         - --proxy-mode=iptables
         - --metrics-bind-address=$(HOST_IP)
         - --healthz-bind-address=$(HOST_IP)
+        {{- if not (eq (int .Values.kubeProxy.conntrackMaxPerCore) 32768) }}
+        - --conntrack-max-per-core={{ .Values.kubeProxy.conntrackMaxPerCore }}
+        {{- end }}
         env:
           - name: NODE_NAME
             valueFrom:

assets/charts/control-plane/kubernetes/values.yaml

@@ -12,6 +12,7 @@ kubeProxy:
   image: k8s.gcr.io/kube-proxy:v1.19.4
   podCIDR: 10.2.0.0/16
   trustedCertsDir: /usr/share/ca-certificates
+  conntrackMaxPerCore: 32768
 kubeScheduler:
   image: k8s.gcr.io/kube-scheduler:v1.19.4
   controlPlaneReplicas: 1

assets/terraform-modules/aws/flatcar-linux/kubernetes/bootkube.tf

@@ -33,4 +33,6 @@ module "bootkube" {
   encrypt_pod_traffic  = var.encrypt_pod_traffic
 
   ignore_x509_cn_check = var.ignore_x509_cn_check
+
+  conntrack_max_per_core = var.conntrack_max_per_core
 }

assets/terraform-modules/aws/flatcar-linux/kubernetes/variables.tf

@@ -196,3 +196,8 @@ variable "ignore_x509_cn_check" {
   type        = bool
   default     = false
 }
+
+variable "conntrack_max_per_core" {
+  description = "--conntrack-max-per-core value for kube-proxy. Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore the conntrack-min kube-proxy flag)."
+  type        = number
+}

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/bootkube.tf

@@ -26,4 +26,6 @@ module "bootkube" {
   encrypt_pod_traffic  = var.encrypt_pod_traffic
 
   ignore_x509_cn_check = var.ignore_x509_cn_check
+
+  conntrack_max_per_core = var.conntrack_max_per_core
 }

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/variables.tf

@@ -195,3 +195,8 @@ variable "ignore_x509_cn_check" {
   type        = bool
   default     = false
 }
+
+variable "conntrack_max_per_core" {
+  description = "--conntrack-max-per-core value for kube-proxy. Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore the conntrack-min kube-proxy flag)."
+  type        = number
+}

assets/terraform-modules/bootkube/assets.tf

@@ -31,26 +31,26 @@ resource "local_file" "bootstrap-scheduler" {
 resource "local_file" "kube-apiserver" {
   filename = "${var.asset_dir}/charts/kube-system/kube-apiserver.yaml"
   content = templatefile("${path.module}/resources/charts/kube-apiserver.yaml", {
-    kube_apiserver_image     = var.container_images["kube_apiserver"]
-    etcd_servers             = join(",", formatlist("https://%s:2379", var.etcd_servers))
-    cloud_provider           = var.cloud_provider
-    service_cidr             = var.service_cidr
-    trusted_certs_dir        = var.trusted_certs_dir
-    ca_cert                  = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
-    apiserver_key            = base64encode(tls_private_key.apiserver.private_key_pem)
-    apiserver_cert           = base64encode(tls_locally_signed_cert.apiserver.cert_pem)
-    serviceaccount_pub       = base64encode(tls_private_key.service-account.public_key_pem)
-    etcd_ca_cert             = base64encode(tls_self_signed_cert.etcd-ca.cert_pem)
-    etcd_client_cert         = base64encode(tls_locally_signed_cert.client.cert_pem)
-    etcd_client_key          = base64encode(tls_private_key.client.private_key_pem)
-    enable_aggregation       = var.enable_aggregation
-    aggregation_ca_cert      = var.enable_aggregation == true ? base64encode(join(" ", tls_self_signed_cert.aggregation-ca.*.cert_pem)) : ""
-    aggregation_client_cert  = var.enable_aggregation == true ? base64encode(join(" ", tls_locally_signed_cert.aggregation-client.*.cert_pem)) : ""
-    aggregation_client_key   = var.enable_aggregation == true ? base64encode(join(" ", tls_private_key.aggregation-client.*.private_key_pem)) : ""
-    replicas                 = length(var.etcd_servers)
-    extra_flags              = var.kube_apiserver_extra_flags
-    enable_tls_bootstrap     = var.enable_tls_bootstrap
-    ignore_x509_cn_check     = var.ignore_x509_cn_check
+    kube_apiserver_image    = var.container_images["kube_apiserver"]
+    etcd_servers            = join(",", formatlist("https://%s:2379", var.etcd_servers))
+    cloud_provider          = var.cloud_provider
+    service_cidr            = var.service_cidr
+    trusted_certs_dir       = var.trusted_certs_dir
+    ca_cert                 = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+    apiserver_key           = base64encode(tls_private_key.apiserver.private_key_pem)
+    apiserver_cert          = base64encode(tls_locally_signed_cert.apiserver.cert_pem)
+    serviceaccount_pub      = base64encode(tls_private_key.service-account.public_key_pem)
+    etcd_ca_cert            = base64encode(tls_self_signed_cert.etcd-ca.cert_pem)
+    etcd_client_cert        = base64encode(tls_locally_signed_cert.client.cert_pem)
+    etcd_client_key         = base64encode(tls_private_key.client.private_key_pem)
+    enable_aggregation      = var.enable_aggregation
+    aggregation_ca_cert     = var.enable_aggregation == true ? base64encode(join(" ", tls_self_signed_cert.aggregation-ca.*.cert_pem)) : ""
+    aggregation_client_cert = var.enable_aggregation == true ? base64encode(join(" ", tls_locally_signed_cert.aggregation-client.*.cert_pem)) : ""
+    aggregation_client_key  = var.enable_aggregation == true ? base64encode(join(" ", tls_private_key.aggregation-client.*.private_key_pem)) : ""
+    replicas                = length(var.etcd_servers)
+    extra_flags             = var.kube_apiserver_extra_flags
+    enable_tls_bootstrap    = var.enable_tls_bootstrap
+    ignore_x509_cn_check    = var.ignore_x509_cn_check
   })
 }
 
@@ -82,6 +82,7 @@ resource "local_file" "kubernetes" {
     serviceaccount_key            = base64encode(tls_private_key.service-account.private_key_pem)
     etcd_endpoints                = var.etcd_endpoints
     enable_tls_bootstrap          = var.enable_tls_bootstrap
+    conntrack_max_per_core        = var.conntrack_max_per_core
   })
 }
 

assets/terraform-modules/bootkube/resources/charts/kubernetes.yaml

@@ -12,6 +12,7 @@ kubeProxy:
   image: ${kube_proxy_image}
   podCIDR: ${pod_cidr}
   trustedCertsDir: ${trusted_certs_dir}
+  conntrackMaxPerCore: ${conntrack_max_per_core}
 kubeScheduler:
   image: ${kube_scheduler_image}
   controlPlaneReplicas: ${control_plane_replicas}

assets/terraform-modules/bootkube/variables.tf

@@ -179,3 +179,8 @@ variable "encrypt_pod_traffic" {
   type        = bool
   default     = false
 }
+
+variable "conntrack_max_per_core" {
+  description = "--conntrack-max-per-core value for kube-proxy. Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore the conntrack-min kube-proxy flag)."
+  type        = number
+}

assets/terraform-modules/packet/flatcar-linux/kubernetes/bootkube.tf

@@ -46,4 +46,6 @@ module "bootkube" {
   encrypt_pod_traffic         = var.encrypt_pod_traffic
 
   ignore_x509_cn_check = var.ignore_x509_cn_check
+
+  conntrack_max_per_core = var.conntrack_max_per_core
 }

assets/terraform-modules/packet/flatcar-linux/kubernetes/variables.tf

@@ -213,3 +213,8 @@ variable "ignore_x509_cn_check" {
   type        = bool
   default     = false
 }
+
+variable "conntrack_max_per_core" {
+  description = "--conntrack-max-per-core value for kube-proxy. Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore the conntrack-min kube-proxy flag)."
+  type        = number
+}

ci/aws/aws-cluster.lokocfg.envsubst

@@ -42,8 +42,9 @@ EOF
     instance_type = "i3.large"
     spot_price    = "0.08"
     labels        = {
-      "testing.io" = "yes",
-      "roleofnode" = "testing",
+      "testing.io"          = "yes",
+      "roleofnode"          = "testing",
+      "conntrack-modified"  = "true",
     }
     tags = {
       "deployment" = "ci"
@@ -62,6 +63,24 @@ storage:
       group:
         id: 500
 EOF
+      ,
+      <<EOF
+storage:
+  files:
+    - path: /etc/modules-load.d/nf.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          nf_conntrack
+    - path: /etc/sysctl.d/nf.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          net.netfilter.nf_conntrack_max=50000
+EOF
+      ,
     ]
   }
 
@@ -72,8 +91,9 @@ EOF
     instance_type = "t2.small"
     spot_price    = "0.01"
     labels        = {
-      "testing.io" = "yes",
-      "roleofnode" = "testing",
+      "testing.io"          = "yes",
+      "roleofnode"          = "testing",
+      "conntrack-modified"  = "true",
     }
     taints        = {
       "nodeType" = "storage:NoSchedule"
@@ -99,13 +119,35 @@ storage:
       group:
         id: 500
 EOF
+      ,
+      <<EOF
+storage:
+  files:
+    - path: /etc/modules-load.d/nf.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          nf_conntrack
+    - path: /etc/sysctl.d/nf.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          net.netfilter.nf_conntrack_max=50000
+EOF
+      ,
     ]
   }
 
   # Adds oidc flags to API server with default values.
   # Acts as a smoke test to check if API server is functional after addition
   # of extra flags.
   oidc {}
+
+  # Disable kube-proxy setting net.netfilter.nf_conntrack_max so we can
+  # set it per worker pool via CLC snippet.
+  conntrack_max_per_core = 0
 }
 
 component "metrics-server" {}

ci/baremetal/baremetal-cluster.lokocfg.envsubst

@@ -40,6 +40,8 @@ cluster "bare-metal" {
     "testing.io" = "yes",
     "roleofnode" = "testing",
   }
+
+  conntrack_max_per_core = 65000
 }
 
 component "inspektor-gadget" {}

ci/packet/packet-cluster.lokocfg.envsubst

@@ -62,6 +62,8 @@ EOF
   oidc {}
 
   ignore_x509_cn_check = true
+
+  conntrack_max_per_core = 65000
 }
 
 component "metrics-server" {}