Merge pull request #76 from getamis/feat/nerdctl

FEATURE: replace docker to nerdctl
getamis · Oct 22, 2023 · 1849824 · 1849824
2 parents 64e6db0 + a5600cc
commit 1849824
Show file tree

Hide file tree

Showing 9 changed files with 37 additions and 16 deletions.
diff --git a/modules/kubelet/files/scripts/init-configs.sh b/modules/kubelet/files/scripts/init-configs.sh
@@ -24,10 +24,14 @@ function require_ev_one() {
 
 source /opt/kubernetes/bin/get-host-info.sh
 
+NERDCTL_BIN_PATH=${NERDCTL_BIN_PATH:="/opt/bin"}
+mkdir -p ${NERDCTL_BIN_PATH}
+sudo tar -xvf /opt/bin/nerdctl.tar.gz -C ${NERDCTL_BIN_PATH}
+
 require_ev_all CFSSL_IMAGE_REPO CFSSL_IMAGE_TAG
 
 CFSSL_IMAGE="${CFSSL_IMAGE_REPO}:${CFSSL_IMAGE_TAG}"
-DOCKER_EXEC="${DOCKER_EXEC:-/usr/bin/docker}"
+NERDCTL_EXEC=${NERDCTL_EXEC:-"/opt/bin/nerdctl"}
 
 KUBE_OPT_PATH=${KUBE_OPT_PATH:="/opt/kubernetes"}
 KUBE_ETC_PATH=${KUBE_ETC_PATH:="/etc/kubernetes"}
@@ -79,7 +83,7 @@ if test -f ${CSR_FILE_SRC} && ! test -f ${KUBELET_VAR_PATH}/pki/${FILE_NAME} ; t
   generate::file ${CA_CONFIG_SRC} ${CA_CONFIG_DEST}
   generate::file ${CSR_FILE_SRC} ${CSR_FILE_DEST}
 
-  ${DOCKER_EXEC} run --rm \
+  ${NERDCTL_EXEC} run --rm \
   -v ${KUBELET_VAR_PATH}/pki/:/tmp/pki/ \
   -v ${KUBE_ETC_PATH}/pki/:${KUBE_ETC_PATH}/pki/ \
   -e HOSTNAME=${HOSTNAME} \

diff --git a/modules/kubelet/files/scripts/kubelet-wrapper.sh b/modules/kubelet/files/scripts/kubelet-wrapper.sh
@@ -22,7 +22,7 @@ PARAMS=("$@")
 [[ ! -z ${PROVIDER_ID} ]] && PARAMS+=(--provider-id=${PROVIDER_ID})
 
 set -x
-exec /usr/bin/docker run --name kubelet \
+exec /opt/bin/nerdctl run --name kubelet \
   --log-driver=journald \
   --privileged \
   --pid host \
@@ -36,12 +36,11 @@ exec /usr/bin/docker run --name kubelet \
   --volume /usr/share/ca-certificates:/usr/share/ca-certificates:ro \
   --volume /var/lib/containerd/:/var/lib/containerd \
   --volume /var/lib/calico:/var/lib/calico:ro \
-  --volume /var/lib/docker:/var/lib/docker \
-  --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
+  --volume /var/lib/kubelet:/var/lib/kubelet:rshared \
   --volume /var/log:/var/log \
-  --volume /var/run/lock:/var/run/lock:z \
-  --volume /opt/cni/bin:/opt/cni/bin:z \
-  --volume /opt/bin/ecr-credential-provider:/opt/bin/ecr-credential-provider:z \
+  --volume /var/run/lock:/var/run/lock \
+  --volume /opt/cni/bin:/opt/cni/bin \
+  --volume /opt/bin/ecr-credential-provider:/opt/bin/ecr-credential-provider \
   --volume /etc/cni/net.d:/etc/cni/net.d \
   ${KUBELET_IMAGE} \
     --node-ip=${HOST_IP} \

diff --git a/modules/kubelet/files/scripts/node-shutdown.sh b/modules/kubelet/files/scripts/node-shutdown.sh
@@ -14,7 +14,7 @@ function require_ev_all() {
 
 function node_cleaning(){
 	#	kubectl cordon node
-	docker run --rm \
+	nerdctl run --rm \
 		-v /etc/kubernetes/kubelet.conf:/root/.kube/config:ro \
 		-v /var/lib/kubelet/pki/kubelet-client-current.pem:/var/lib/kubelet/pki/kubelet-client-current.pem:ro \
 		--entrypoint=kubectl "${KUBECTL_IMAGE}" cordon "${HOSTNAME_FQDN}"
@@ -23,7 +23,7 @@ function node_cleaning(){
 	systemctl stop kubelet.service
 
 	# kubectl delete node
-	docker run --rm \
+	nerdctl run --rm \
 		-v /etc/kubernetes/kubelet.conf:/root/.kube/config:ro \
 		-v /var/lib/kubelet/pki/kubelet-client-current.pem:/var/lib/kubelet/pki/kubelet-client-current.pem:ro \
 		--entrypoint=kubectl "${KUBECTL_IMAGE}" delete node "${HOSTNAME_FQDN}"

diff --git a/modules/kubelet/main.tf b/modules/kubelet/main.tf
@@ -40,6 +40,17 @@ data "ignition_file" "envsubst" {
   }
 }
 
+data "ignition_file" "nerdctl" {
+  path      = "/opt/bin/nerdctl.tar.gz"
+  mode      = 500
+  overwrite = true
+
+  source {
+    source       = local.binaries["nerdctl"].source
+    verification = local.binaries["nerdctl"].checksum
+  }
+}
+
 data "ignition_file" "kubernetes_env" {
   path      = "/etc/default/kubernetes.env"
   mode      = 420

diff --git a/modules/kubelet/outputs.tf b/modules/kubelet/outputs.tf
@@ -11,6 +11,7 @@ output "files" {
     [
       data.ignition_file.cni_plugin_tgz.rendered,
       data.ignition_file.envsubst.rendered,
+      data.ignition_file.nerdctl.rendered,
       data.ignition_file.kubernetes_env.rendered,
       data.ignition_file.init_configs_sh.rendered,
       data.ignition_file.get_host_info_sh.rendered,

diff --git a/modules/kubelet/templates/services/10-kubelet.conf.tpl b/modules/kubelet/templates/services/10-kubelet.conf.tpl
@@ -5,6 +5,6 @@ Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
 EnvironmentFile=-/etc/default/kubernetes.env
 EnvironmentFile=-/var/lib/kubelet/kubelet-flags.env
 ExecStart=
-ExecStartPre=-/bin/docker rm kubelet
+ExecStartPre=-/opt/bin/nerdctl rm kubelet
 ExecStart=systemd-inhibit --what=shutdown --mode=delay /opt/kubernetes/bin/kubelet-wrapper $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_NETWORK_ARGS $KUBELET_CLOUD_PROVIDER_ARGS $KUBELET_EXTRA_ARGS
-ExecStop=/bin/bash -c "docker stop kubelet && crictl stop -t 60 $$(crictl ps -q)"
+ExecStop=/bin/bash -c "nerdctl stop kubelet && crictl stop -t 60 $$(crictl ps -q)"
diff --git a/modules/kubelet/variables_defaults.tf b/modules/kubelet/variables_defaults.tf
@@ -10,6 +10,12 @@ locals {
         source   = "https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-Linux-x86_64"
         checksum = "sha512-91dfd502ab14173ac8af35ca318c9872ec3e0b04b34580b65f787faead355e29ca9609aaeb6ca0629d7dd9cfaeaa83769a166eb03923ae19441da04150e865c6"
       }
+    },
+    {
+      nerdctl = {
+        source   = "https://github.com/containerd/nerdctl/releases/download/v1.6.0/nerdctl-1.6.0-linux-amd64.tar.gz"
+        checksum = "sha512-89dcba32badfd1481d88cd5f4179ff99348578af5004a7e96daa05101e99ba7448685596692ada3186f718ffd1166768ac6a22e041c5887e416e6dc7fda97f24"
+      }
   }, var.binaries)
 
   containers = merge({

diff --git a/scripts/init-addons.sh b/scripts/init-addons.sh
@@ -21,11 +21,11 @@ source /opt/kubernetes/bin/get-host-info.sh
 export KUBECONFIG=/etc/kubernetes/admin.conf
 
 set -x
-docker run --rm \
+nerdctl run --rm \
   -v /etc/kubernetes/admin.conf:/root/.kube/config:ro \
   --entrypoint=kubectl ${KUBECTL_IMAGE} label node ${HOSTNAME_FQDN} node-role.kubernetes.io/master="" --overwrite
 
-docker run --rm \
+nerdctl run --rm \
   -v /etc/kubernetes/admin.conf:/root/.kube/config:ro \
   -v ${ADDONS_PATH}:${ADDONS_PATH}:ro \
   --entrypoint=kubectl ${KUBECTL_IMAGE} apply -f ${ADDONS_PATH}
diff --git a/variables.tf b/variables.tf
@@ -107,8 +107,8 @@ variable "tls_bootstrap_token" {
 
 variable "cloud_provider" {
   description = "The cloud provider configuration."
-  type = string
-  default = ""
+  type        = string
+  default     = ""
 }
 
 variable "ccm_config" {