DEPRECATE: replace aws-calico by aws-network-policy-agent (#82)

- revert: #76 - nerdctl 會偷塞 conflist 到 /etc/cni/net.d 底下，導致早於 aws-cni 啟動的 pod 會用到 nerdctl 的 config - replace aws-calico by aws-network-policy-agent
getamis · Jan 8, 2024 · 8f0a741 · 8f0a741
1 parent 4c0d5c7
commit 8f0a741
Show file tree

Hide file tree

Showing 19 changed files with 324 additions and 749 deletions.
diff --git a/docs/variables/kubelet.md b/docs/variables/kubelet.md
@@ -34,7 +34,6 @@ No modules.
 | [ignition_file.kubelet_wrapper_sh](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.kubernetes_env](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.logind_kubelet_conf](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
-| [ignition_file.nerdctl](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.node_shutdown_sh](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.sysctl_k8s_conf](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.sysctl_max_user_watches_conf](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |

diff --git a/docs/variables/master.md b/docs/variables/master.md
@@ -38,7 +38,7 @@ This document gives an overview of variables used in the Ignition of the Kuberne
 | [ignition_file.apiserver_kubelet_client_key](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.audit_log_policy](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.aws_cloud_controller_manager](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
-| [ignition_file.aws_cni_calico_yaml](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
+| [ignition_file.aws_network_policy_controller_yaml](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.aws_vpc_cni_yaml](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.bootstrap_token_rbac](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
 | [ignition_file.bootstrap_token_secret](https://registry.terraform.io/providers/community-terraform-providers/ignition/2.1.2/docs/data-sources/file) | data source |
@@ -83,10 +83,10 @@ This document gives an overview of variables used in the Ignition of the Kuberne
 | <a name="input_containers"></a> [containers](#input\_containers) | Desired containers(kube-apiserver, kube-controller-manager, cfssl, coredns, and so on) repo and tag. | <pre>map(object({<br>    repo = string<br>    tag  = string<br>  }))</pre> | `{}` | no |
 | <a name="input_controller_manager_flags"></a> [controller\_manager\_flags](#input\_controller\_manager\_flags) | The flags of kube-controller-manager. The variables need to follow https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/. Do not use underline. | `map(any)` | `{}` | no |
 | <a name="input_coredns_config"></a> [coredns\_config](#input\_coredns\_config) | The configuration of CoreDNS. | `map(any)` | `{}` | no |
-| <a name="input_enable_calico"></a> [enable\_calico](#input\_enable\_calico) | (optional) whether enable the calico | `bool` | `false` | no |
 | <a name="input_enable_eni_prefix"></a> [enable\_eni\_prefix](#input\_enable\_eni\_prefix) | (Optional) assign prefix to AWS EC2 network interface | `bool` | `true` | no |
 | <a name="input_enable_iam_auth"></a> [enable\_iam\_auth](#input\_enable\_iam\_auth) | Enable AWS IAM authenticator or not. | `bool` | `false` | no |
 | <a name="input_enable_irsa"></a> [enable\_irsa](#input\_enable\_irsa) | Enable AWS IAM role service account or not. | `bool` | `false` | no |
+| <a name="input_enable_network_policy"></a> [enable\_network\_policy](#input\_enable\_network\_policy) | (Optional) [AWS VPC CNI] Enable AWS Network Policy Agent. | `bool` | `false` | no |
 | <a name="input_encryption_secret"></a> [encryption\_secret](#input\_encryption\_secret) | The secret key for encrypting | `string` | `""` | no |
 | <a name="input_etcd_endpoints"></a> [etcd\_endpoints](#input\_etcd\_endpoints) | The comma separated list of etcd endpoints (e.g., 'http://etcd1:2379,http://etcd2:2379'). | `string` | `""` | no |
 | <a name="input_external_snat"></a> [external\_snat](#input\_external\_snat) | (Optional) [AWS VPC CNI] Specifies whether an external NAT gateway should be used to provide SNAT of secondary ENI IP addresses. If set to true, the SNAT iptables rule and off-VPC IP rule are not applied, and these rules are removed if they have already been applied. | `bool` | `false` | no |

diff --git a/modules/kubelet/files/scripts/init-configs.sh b/modules/kubelet/files/scripts/init-configs.sh
@@ -24,14 +24,11 @@ function require_ev_one() {
 
 source /opt/kubernetes/bin/get-host-info.sh
 
-NERDCTL_BIN_PATH=${NERDCTL_BIN_PATH:="/opt/bin"}
-mkdir -p ${NERDCTL_BIN_PATH}
-sudo tar -xvf /opt/bin/nerdctl.tar.gz -C ${NERDCTL_BIN_PATH}
 
 require_ev_all CFSSL_IMAGE_REPO CFSSL_IMAGE_TAG
 
 CFSSL_IMAGE="${CFSSL_IMAGE_REPO}:${CFSSL_IMAGE_TAG}"
-NERDCTL_EXEC=${NERDCTL_EXEC:-"/opt/bin/nerdctl"}
+DOCKER_EXEC="${DOCKER_EXEC:-/usr/bin/docker}"
 
 KUBE_OPT_PATH=${KUBE_OPT_PATH:="/opt/kubernetes"}
 KUBE_ETC_PATH=${KUBE_ETC_PATH:="/etc/kubernetes"}
@@ -83,7 +80,7 @@ if test -f ${CSR_FILE_SRC} && ! test -f ${KUBELET_VAR_PATH}/pki/${FILE_NAME} ; t
   generate::file ${CA_CONFIG_SRC} ${CA_CONFIG_DEST}
   generate::file ${CSR_FILE_SRC} ${CSR_FILE_DEST}
 
-  ${NERDCTL_EXEC} run --rm \
+  ${DOCKER_EXEC} run --rm \
   -v ${KUBELET_VAR_PATH}/pki/:/tmp/pki/ \
   -v ${KUBE_ETC_PATH}/pki/:${KUBE_ETC_PATH}/pki/ \
   -e HOSTNAME=${HOSTNAME} \

diff --git a/modules/kubelet/files/scripts/kubelet-wrapper.sh b/modules/kubelet/files/scripts/kubelet-wrapper.sh
@@ -22,7 +22,7 @@ PARAMS=("$@")
 [[ ! -z ${PROVIDER_ID} ]] && PARAMS+=(--provider-id=${PROVIDER_ID})
 
 set -x
-exec /opt/bin/nerdctl run --name kubelet \
+exec /usr/bin/docker run --name kubelet \
   --log-driver=journald \
   --privileged \
   --pid host \
@@ -35,12 +35,12 @@ exec /opt/bin/nerdctl run --name kubelet \
   --volume /sys/fs/cgroup:/sys/fs/cgroup \
   --volume /usr/share/ca-certificates:/usr/share/ca-certificates:ro \
   --volume /var/lib/containerd/:/var/lib/containerd \
-  --volume /var/lib/calico:/var/lib/calico:ro \
-  --volume /var/lib/kubelet:/var/lib/kubelet:rshared \
+  --volume /var/lib/docker:/var/lib/docker \
+  --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
   --volume /var/log:/var/log \
-  --volume /var/run/lock:/var/run/lock \
-  --volume /opt/cni/bin:/opt/cni/bin \
-  --volume /opt/bin/ecr-credential-provider:/opt/bin/ecr-credential-provider \
+  --volume /var/run/lock:/var/run/lock:z \
+  --volume /opt/cni/bin:/opt/cni/bin:z \
+  --volume /opt/bin/ecr-credential-provider:/opt/bin/ecr-credential-provider:z \
   --volume /etc/cni/net.d:/etc/cni/net.d \
   ${KUBELET_IMAGE} \
     --node-ip=${HOST_IP} \

diff --git a/modules/kubelet/files/scripts/node-shutdown.sh b/modules/kubelet/files/scripts/node-shutdown.sh
@@ -14,7 +14,7 @@ function require_ev_all() {
 
 function node_cleaning(){
 	#	kubectl cordon node
-	nerdctl run --network host --rm \
+	docker run --network host --rm \
 		-v /etc/kubernetes/kubelet.conf:/root/.kube/config:ro \
 		-v /var/lib/kubelet/pki/kubelet-client-current.pem:/var/lib/kubelet/pki/kubelet-client-current.pem:ro \
 		--entrypoint=kubectl "${KUBECTL_IMAGE}" cordon "${HOSTNAME_FQDN}"
@@ -23,7 +23,7 @@ function node_cleaning(){
 	systemctl stop kubelet.service
 
 	# kubectl delete node
-	nerdctl run --network host --rm \
+	docker run --network host --rm \
 		-v /etc/kubernetes/kubelet.conf:/root/.kube/config:ro \
 		-v /var/lib/kubelet/pki/kubelet-client-current.pem:/var/lib/kubelet/pki/kubelet-client-current.pem:ro \
 		--entrypoint=kubectl "${KUBECTL_IMAGE}" delete node "${HOSTNAME_FQDN}"

diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf
@@ -40,16 +40,6 @@ data "ignition_file" "envsubst" {
   }
 }
 
-data "ignition_file" "nerdctl" {
-  path      = "/opt/bin/nerdctl.tar.gz"
-  mode      = 500
-  overwrite = true
-
-  source {
-    source       = local.binaries["nerdctl"].source
-    verification = local.binaries["nerdctl"].checksum
-  }
-}
 
 data "ignition_file" "kubernetes_env" {
   path      = "/etc/default/kubernetes.env"

diff --git a/modules/kubelet/outputs.tf b/modules/kubelet/outputs.tf
@@ -11,7 +11,6 @@ output "files" {
     [
       data.ignition_file.cni_plugin_tgz.rendered,
       data.ignition_file.envsubst.rendered,
-      data.ignition_file.nerdctl.rendered,
       data.ignition_file.kubernetes_env.rendered,
       data.ignition_file.init_configs_sh.rendered,
       data.ignition_file.get_host_info_sh.rendered,

diff --git a/modules/kubelet/templates/services/10-kubelet.conf.tpl b/modules/kubelet/templates/services/10-kubelet.conf.tpl
@@ -5,6 +5,6 @@ Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
 EnvironmentFile=-/etc/default/kubernetes.env
 EnvironmentFile=-/var/lib/kubelet/kubelet-flags.env
 ExecStart=
-ExecStartPre=-/opt/bin/nerdctl rm kubelet
+ExecStartPre=-/bin/docker rm kubelet
 ExecStart=systemd-inhibit --what=shutdown --mode=delay /opt/kubernetes/bin/kubelet-wrapper $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_NETWORK_ARGS $KUBELET_CLOUD_PROVIDER_ARGS $KUBELET_EXTRA_ARGS
-ExecStop=/bin/bash -c "nerdctl stop kubelet && crictl stop -t 60 $$(crictl ps -q)"
+ExecStop=/bin/bash -c "docker stop kubelet && crictl stop -t 60 $$(crictl ps -q)"
diff --git a/modules/kubelet/variables_defaults.tf b/modules/kubelet/variables_defaults.tf
@@ -10,13 +10,7 @@ locals {
         source   = "https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-Linux-x86_64"
         checksum = "sha512-91dfd502ab14173ac8af35ca318c9872ec3e0b04b34580b65f787faead355e29ca9609aaeb6ca0629d7dd9cfaeaa83769a166eb03923ae19441da04150e865c6"
       }
-    },
-    {
-      nerdctl = {
-        source   = "https://github.com/containerd/nerdctl/releases/download/v1.6.0/nerdctl-1.6.0-linux-amd64.tar.gz"
-        checksum = "sha512-89dcba32badfd1481d88cd5f4179ff99348578af5004a7e96daa05101e99ba7448685596692ada3186f718ffd1166768ac6a22e041c5887e416e6dc7fda97f24"
-      }
-  }, var.binaries)
+    }, var.binaries)
 
   containers = merge({
     kubelet = {

diff --git a/network-plugins.tf b/network-plugins.tf
@@ -12,29 +12,28 @@ data "ignition_file" "aws_vpc_cni_yaml" {
       init_image       = "${local.containers["vpc_cni_init"].repo}:${local.containers["vpc_cni_init"].tag}"
       node_agent_image = "${local.containers["aws_network_policy_agent"].repo}:${local.containers["aws_network_policy_agent"].tag}"
       # vpc cni config
-      annotate_pod_ip   = var.annotate_pod_ip
-      cni_version       = local.containers["vpc_cni"].tag
-      cluster_endpoint  = var.internal_endpoint
-      enable_eni_prefix = var.enable_eni_prefix
-      external_snat     = var.external_snat
-      log_level         = var.log_level["aws_vpc_cni"]
+      annotate_pod_ip       = var.annotate_pod_ip
+      cni_version           = local.containers["vpc_cni"].tag
+      cluster_endpoint      = var.internal_endpoint
+      enable_eni_prefix     = var.enable_eni_prefix
+      enable_network_policy = var.enable_network_policy
+      external_snat         = var.external_snat
+      log_level             = var.log_level["aws_vpc_cni"]
     })
     mime = "text/yaml"
   }
 }
 
-data "ignition_file" "aws_cni_calico_yaml" {
-  count = var.enable_calico ? 1 : 0
+data "ignition_file" "aws_network_policy_controller_yaml" {
+  count = (var.network_plugin == "amazon-vpc" && var.enable_network_policy) ? 1 : 0
 
   mode      = 420
-  path      = "${local.etc_path}/addons/calico-cni.yaml"
+  path      = "${local.etc_path}/addons/aws-network-policy-controller.yaml"
   overwrite = true
 
   content {
-    content = templatefile("${path.module}/templates/network-plugins/amazon-vpc/calico.yaml.tpl", {
-      node_image       = "${local.containers["calico_node"].repo}:${local.containers["calico_node"].tag}"
-      typha_image      = "${local.containers["calico_typha"].repo}:${local.containers["calico_typha"].tag}"
-      autoscaler_image = "${local.containers["calico_autoscaler"].repo}:${local.containers["calico_autoscaler"].tag}"
+    content = templatefile("${path.module}/templates/network-plugins/amazon-vpc/aws-network-policy-controller.yaml.tpl", {
+      image            = "${local.containers["aws_network_policy_controller"].repo}:${local.containers["aws_network_policy_controller"].tag}"
     })
     mime = "text/yaml"
   }

diff --git a/outputs.tf b/outputs.tf
@@ -27,6 +27,9 @@ output "files" {
       data.ignition_file.kube_proxy_cm.rendered,
       data.ignition_file.aws_vpc_cni_yaml[0].rendered,
     ] : [],
+    var.network_plugin == "amazon-vpc" && var.enable_network_policy ? [
+      data.ignition_file.aws_network_policy_controller_yaml[0].rendered,
+    ] : [],
     var.network_plugin == "flannel" ? [
       data.ignition_file.kube_proxy.rendered,
       data.ignition_file.kube_proxy_cm.rendered,
@@ -35,9 +38,6 @@ output "files" {
     var.network_plugin == "cilium-vxlan" ? [
       data.ignition_file.cilium_vxlan_yaml[0].rendered,
     ] : [],
-    var.enable_calico ? [
-      data.ignition_file.aws_cni_calico_yaml[0].rendered,
-    ] : [],
     module.kubelet.files,
     module.admin_kubeconfig.files,
     module.controller_manager_kubeconfig.files,

diff --git a/scripts/init-addons.sh b/scripts/init-addons.sh
@@ -21,11 +21,11 @@ source /opt/kubernetes/bin/get-host-info.sh
 export KUBECONFIG=/etc/kubernetes/admin.conf
 
 set -x
-nerdctl run --network host --rm \
+docker run --network host --rm \
   -v /etc/kubernetes/admin.conf:/root/.kube/config:ro \
   --entrypoint=kubectl ${KUBECTL_IMAGE} label node ${HOSTNAME_FQDN} node-role.kubernetes.io/master="" --overwrite
 
-nerdctl run --network host --rm \
+docker run --network host --rm \
   -v /etc/kubernetes/admin.conf:/root/.kube/config:ro \
   -v ${ADDONS_PATH}:${ADDONS_PATH}:ro \
   --entrypoint=kubectl ${KUBECTL_IMAGE} apply -f ${ADDONS_PATH}
diff --git a/templates/addons/coredns.yaml.tpl b/templates/addons/coredns.yaml.tpl
@@ -88,6 +88,7 @@ metadata:
   name: coredns
   namespace: kube-system
   labels:
+    # WARN: DO NOT change these lables, it may break the kube-dns service.
     k8s-app: kube-dns
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
@@ -220,6 +221,7 @@ metadata:
     kubernetes.io/name: "CoreDNS"
 spec:
   selector:
+    # WARN: DO NOT change these lables, it may break the kube-dns service.
     k8s-app: kube-dns
   clusterIP: ${cluster_dns_ip}
   ports:

diff --git a/templates/network-plugins/amazon-vpc/README.md b/templates/network-plugins/amazon-vpc/README.md
@@ -0,0 +1,16 @@
+# How to generate the manifests?
+
+### Amazon Network Policy Controller
+1. git clone https://github.com/aws/amazon-network-policy-controller-k8s
+1. Add generate-manefest to makefile
+   ```
+   .PHONY: generate-manifests
+   generate-manifests: manifests kustomize
+       cd config/controller && $(KUSTOMIZE) edit set image controller=${IMG}
+       $(KUSTOMIZE) build config/default > config/amazon-network-policy-controller-k8s.yaml
+   ```
+1. `make generate-manifests`
+1. Copy `config/amazon-network-policy-controller-k8s.yaml`
+
+### AWS VPC CNI
+1. Vendored from https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.16.0/config/master/aws-k8s-cni.yaml