Merge pull request #2111 from chilianyi/support-cri-docker

Support use docker for version >= v1.24.0

cmd/kk/apis/kubekey/v1alpha2/default.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
 )
 
 const (
@@ -42,6 +43,7 @@ const (
 	DefaultEtcdVersion             = "v3.5.6"
 	DefaultEtcdPort                = "2379"
 	DefaultDockerVersion           = "24.0.6"
+	DefaultCriDockerdVersion        = "0.3.9"
 	DefaultContainerdVersion       = "1.7.12"
 	DefaultRuncVersion             = "v1.1.11"
 	DefaultCrictlVersion           = "v1.29.0"
@@ -68,6 +70,7 @@ const (
 	DefaultProxyMode               = "ipvs"
 	DefaultCrioEndpoint            = "unix:///var/run/crio/crio.sock"
 	DefaultContainerdEndpoint      = "unix:///run/containerd/containerd.sock"
+	DefaultCriDockerdEndpoint      = "unix:///var/run/cri-dockerd.sock"
 	DefaultIsulaEndpoint           = "unix:///var/run/isulad.sock"
 	Etcd                           = "etcd"
 	Master                         = "master"
@@ -315,7 +318,11 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
 	if cfg.Kubernetes.ContainerRuntimeEndpoint == "" {
 		switch cfg.Kubernetes.ContainerManager {
 		case Docker:
-			cfg.Kubernetes.ContainerRuntimeEndpoint = ""
+			if kubernetes.IsAtLeastV124(cfg.Kubernetes.Version){
+				cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCriDockerdEndpoint
+			} else {
+				cfg.Kubernetes.ContainerRuntimeEndpoint = ""
+			}
 		case Crio:
 			cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCrioEndpoint
 		case Containerd:

cmd/kk/pkg/binaries/kubernetes.go

@@ -20,14 +20,14 @@ import (
 	"fmt"
 	"os/exec"
 
-	"github.com/pkg/errors"
-
 	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
+	"github.com/pkg/errors"
 )
 
 // K8sFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them.
@@ -40,6 +40,7 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 	kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand)
 	helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand)
 	docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand)
+	criDockerd := files.NewKubeBinary("cri-dockerd", arch, kubekeyapiv1alpha2.DefaultCriDockerdVersion, path, kubeConf.Arg.DownloadCommand)
 	crictl := files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand)
 	containerd := files.NewKubeBinary("containerd", arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand)
 	runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand)
@@ -49,6 +50,9 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 
 	if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Docker {
 		binaries = append(binaries, docker)
+		if kubernetes.IsAtLeastV124(kubeConf.Cluster.Kubernetes.Version) && kubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
+			binaries = append(binaries, criDockerd)
+		}
 	} else if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Containerd {
 		binaries = append(binaries, containerd, runc)
 	}

cmd/kk/pkg/bootstrap/confirm/tasks.go

@@ -23,16 +23,16 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/mitchellh/mapstructure"
-	"github.com/modood/table"
-	"github.com/pkg/errors"
-	versionutil "k8s.io/apimachinery/pkg/util/version"
-
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
+	"github.com/mitchellh/mapstructure"
+	"github.com/modood/table"
+	"github.com/pkg/errors"
+	versionutil "k8s.io/apimachinery/pkg/util/version"
 )
 
 // PreCheckResults defines the items to be checked.
@@ -107,18 +107,17 @@ func (i *InstallationConfirm) Execute(runtime connector.Runtime) error {
 	fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations")
 	fmt.Println("")
 
-	if k8sVersion, err := versionutil.ParseGeneric(i.KubeConf.Cluster.Kubernetes.Version); err == nil {
-		if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
-			fmt.Println("[Notice]")
-			fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.")
-			fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.")
-			fmt.Println("For more information, see:")
-			fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md")
-			fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes")
-			fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/")
-			fmt.Println("")
-			stopFlag = true
-		}
+	if kubernetes.IsAtLeastV124(i.KubeConf.Cluster.Kubernetes.Version) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker &&
+		i.KubeConf.Cluster.Kubernetes.Type != common.Kubernetes {
+		fmt.Println("[Notice]")
+		fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.")
+		fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.")
+		fmt.Println("For more information, see:")
+		fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md")
+		fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes")
+		fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/")
+		fmt.Println("")
+		stopFlag = true
 	}
 
 	if stopFlag {

cmd/kk/pkg/common/common.go

@@ -62,6 +62,7 @@ const (
 	Hybridnet = "hybridnet"
 
 	Docker     = "docker"
+	CriDockerd = "cri-dockerd"
 	Crictl     = "crictl"
 	Containerd = "containerd"
 	Crio       = "crio"

cmd/kk/pkg/container/docker.go

@@ -21,14 +21,14 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
+	"github.com/pkg/errors"
 )
 
 type SyncDockerBinaries struct {
@@ -64,6 +64,39 @@ func (s *SyncDockerBinaries) Execute(runtime connector.Runtime) error {
 	return nil
 }
 
+type SyncCriDockerdBinaries struct {
+	common.KubeAction
+}
+
+func (s *SyncCriDockerdBinaries) Execute(runtime connector.Runtime) error {
+	if err := utils.ResetTmpDir(runtime); err != nil {
+		return err
+	}
+
+	binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch())
+	if !ok {
+		return errors.New("get KubeBinary by pipeline cache failed")
+	}
+	binariesMap := binariesMapObj.(map[string]*files.KubeBinary)
+
+	criDockerd, ok := binariesMap[common.CriDockerd]
+	if !ok {
+		return errors.New("get KubeBinary key cri-dockerd by pipeline cache failed")
+	}
+
+	dst := filepath.Join(common.TmpDir, criDockerd.FileName)
+	if err := runtime.GetRunner().Scp(criDockerd.Path(), dst); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync cri-dockerd binaries failed"))
+	}
+
+	if _, err := runtime.GetRunner().SudoCmd(
+		fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s && mv cri-dockerd/* /usr/bin && rm -rf cri-dockerd", dst),
+		false); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install container runtime cri-dockerd binaries failed"))
+	}
+	return nil
+}
+
 type EnableContainerdForDocker struct {
 	common.KubeAction
 }
@@ -90,6 +123,19 @@ func (e *EnableDocker) Execute(runtime connector.Runtime) error {
 	return nil
 }
 
+type EnableCriDockerd struct {
+	common.KubeAction
+}
+
+func (e *EnableCriDockerd) Execute(runtime connector.Runtime) error {
+	if _, err := runtime.GetRunner().SudoCmd(
+		"systemctl daemon-reload && systemctl enable cri-docker && systemctl start cri-docker",
+		false); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("enable and start cri-docker failed"))
+	}
+	return nil
+}
+
 type DockerLoginRegistry struct {
 	common.KubeAction
 }
@@ -141,6 +187,15 @@ func (d *DisableDocker) Execute(runtime connector.Runtime) error {
 		filepath.Join("/etc/systemd/system", templates.DockerService.Name()),
 		filepath.Join("/etc/docker", templates.DockerConfig.Name()),
 	}
+
+	if kubernetes.IsAtLeastV124(d.KubeConf.Cluster.Kubernetes.Version) && d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
+		if _, err := runtime.GetRunner().SudoCmd("systemctl disable cri-docker && systemctl stop cri-docker",
+			false); err != nil {
+			return errors.Wrap(errors.WithStack(err), fmt.Sprintf("disable and stop cri-docker failed"))
+		}
+		files = append(files, filepath.Join("/etc/systemd/system", templates.CriDockerService.Name()))
+	}
+
 	if d.KubeConf.Cluster.Registry.DataRoot != "" {
 		files = append(files, d.KubeConf.Cluster.Registry.DataRoot)
 	} else {

cmd/kk/pkg/container/module.go

@@ -30,6 +30,7 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry"
+	versionk8s "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
 )
 
 type InstallContainerModule struct {
@@ -73,6 +74,19 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
 		Retry:    2,
 	}
 
+	syncCriDockerdBinaries := &task.RemoteTask{
+		Name:  "SyncCriDockerdBinaries",
+		Desc:  "Sync cri-dockerd binaries",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true},
+			&CriDockerdExist{Not: true},
+		},
+		Action:   new(SyncCriDockerdBinaries),
+		Parallel: true,
+		Retry:    2,
+	}
+
 	generateContainerdService := &task.RemoteTask{
 		Name:  "GenerateContainerdService",
 		Desc:  "Generate containerd service",
@@ -161,6 +175,48 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
 		Parallel: true,
 	}
 
+	generateCriDockerdService := &task.RemoteTask{
+		Name:  "GenerateCriDockerdService",
+		Desc:  "Generate cri-dockerd service",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true},
+			&CriDockerdExist{Not: true},
+		},
+		Action: &action.Template{
+			Template: templates.CriDockerService,
+			Dst:      filepath.Join("/etc/systemd/system", templates.CriDockerService.Name()),
+		},
+		Parallel: true,
+	}
+
+	enableCriDockerd := &task.RemoteTask{
+		Name:  "EnableCriDockerd",
+		Desc:  "Enable cri-dockerd",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true},
+			&CriDockerdExist{Not: true},
+		},
+		Action:   new(EnableCriDockerd),
+		Parallel: true,
+	}
+
+	if versionk8s.IsAtLeastV124(m.KubeConf.Cluster.Kubernetes.Version) && m.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
+		return []task.Interface{
+			syncBinaries,
+			syncCriDockerdBinaries,
+			generateContainerdService,
+			generateDockerService,
+			generateDockerConfig,
+			enableContainerdForDocker,
+			enableDocker,
+			dockerLoginRegistry,
+			generateCriDockerdService,
+			enableCriDockerd,
+		}
+	}
+
 	return []task.Interface{
 		syncBinaries,
 		generateContainerdService,

cmd/kk/pkg/container/prepares.go

@@ -41,6 +41,24 @@ func (d *DockerExist) PreCheck(runtime connector.Runtime) (bool, error) {
 	return !d.Not, nil
 }
 
+type CriDockerdExist struct {
+	common.KubePrepare
+	Not bool
+}
+
+func (d *CriDockerdExist) PreCheck(runtime connector.Runtime) (bool, error) {
+	output, err := runtime.GetRunner().SudoCmd("if [ -z $(command -v cri-dockerd) ] || [ ! -e /var/run/cri-dockerd.sock ]; "+
+		"then echo 'not exist'; "+
+		"fi", false)
+	if err != nil {
+		return false, err
+	}
+	if strings.Contains(output, "not exist") {
+		return d.Not, nil
+	}
+	return !d.Not, nil
+}
+
 type CrictlExist struct {
 	common.KubePrepare
 	Not bool

cmd/kk/pkg/container/templates/cri_dockerd_service.go

@@ -0,0 +1,63 @@
+/*
+ Copyright 2021 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package templates
+
+import (
+	"text/template"
+
+	"github.com/lithammer/dedent"
+)
+
+var CriDockerService = template.Must(template.New("cri-docker.service").Parse(
+	dedent.Dedent(`[Unit]
+Description=CRI Interface for Docker Application Container Engine
+Documentation=https://docs.mirantis.com
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image docker.io/kubesphere/pause:3.8
+ExecReload=/bin/kill -s HUP $MAINPID
+TimeoutSec=0
+RestartSec=2
+Restart=always
+
+# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
+# Both the old, and new location are accepted by systemd 229 and up, so using the old location
+# to make them work for either version of systemd.
+StartLimitBurst=3
+
+# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
+# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
+# this option work for either version of systemd.
+StartLimitInterval=60s
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Comment TasksMax if your systemd version does not support it.
+# Only systemd 226 and above support this option.
+TasksMax=infinity
+Delegate=yes
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target
+
+    `)))