Feature: k0s runtime remain work (#1686)

* feat: add k0s runtime join/delete/reset/upgrade. Signed-off-by: starComingup <[email protected]> * resolved: log format, cmd in code, simplify code. Signed-off-by: starComingup <[email protected]> * resolved function name confuse, simplify code. Signed-off-by: starComingup <[email protected]> * add runtime entrance, diff k0s、k8s、k3s。 Signed-off-by: starComingup <[email protected]> * resolve conversation and enhanced runtime logic Signed-off-by: starComingup <[email protected]> Signed-off-by: starComingup <[email protected]>
sealerio · Sep 15, 2022 · d1a3e78 · d1a3e78
1 parent 7291ac8
commit d1a3e78
Show file tree

Hide file tree

Showing 19 changed files with 825 additions and 75 deletions.
diff --git a/apply/driver/local.go b/apply/driver/local.go
@@ -18,18 +18,15 @@ import (
 	"fmt"
 	"net"
 
-	imagecommon "github.com/sealerio/sealer/pkg/define/options"
-
-	"github.com/sealerio/sealer/pkg/auth"
-	"github.com/sealerio/sealer/pkg/imageengine"
-
 	"github.com/sealerio/sealer/apply/processor"
 	"github.com/sealerio/sealer/common"
+	"github.com/sealerio/sealer/pkg/auth"
 	"github.com/sealerio/sealer/pkg/client/k8s"
 	"github.com/sealerio/sealer/pkg/clusterfile"
+	imagecommon "github.com/sealerio/sealer/pkg/define/options"
 	"github.com/sealerio/sealer/pkg/filesystem/clusterimage"
+	"github.com/sealerio/sealer/pkg/imageengine"
 	"github.com/sealerio/sealer/pkg/runtime"
-	"github.com/sealerio/sealer/pkg/runtime/kubernetes"
 	v2 "github.com/sealerio/sealer/types/api/v2"
 	"github.com/sealerio/sealer/utils"
 	osi "github.com/sealerio/sealer/utils/os"
@@ -237,7 +234,7 @@ func (applier *Applier) Upgrade(upgradeImgName string) error {
 }
 
 func (applier *Applier) upgrade() error {
-	runtimeInterface, err := kubernetes.NewDefaultRuntime(applier.ClusterDesired, applier.ClusterFile.GetKubeadmConfig())
+	runtimeInterface, err := processor.ChooseRuntime(platform.DefaultMountClusterImageDir(applier.ClusterDesired.Name), applier.ClusterDesired, applier.ClusterFile.GetKubeadmConfig())
 	if err != nil {
 		return fmt.Errorf("failed to init runtime: %v", err)
 	}

diff --git a/apply/processor/create.go b/apply/processor/create.go
@@ -30,7 +30,6 @@ import (
 	"github.com/sealerio/sealer/pkg/guest"
 	"github.com/sealerio/sealer/pkg/plugin"
 	"github.com/sealerio/sealer/pkg/runtime"
-	"github.com/sealerio/sealer/pkg/runtime/kubernetes"
 	v2 "github.com/sealerio/sealer/types/api/v2"
 	"github.com/sealerio/sealer/utils/net"
 	"github.com/sealerio/sealer/utils/platform"
@@ -111,7 +110,8 @@ func (c *CreateProcessor) MountImage(cluster *v2.Cluster) error {
 	if err = c.cloudImageMounter.MountImage(cluster); err != nil {
 		return err
 	}
-	runTime, err := kubernetes.NewDefaultRuntime(cluster, c.ClusterFile.GetKubeadmConfig())
+	//TODO split kubeadm config from cluster file.
+	runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, c.ClusterFile.GetKubeadmConfig())
 	if err != nil {
 		return fmt.Errorf("failed to init runtime: %v", err)
 	}

diff --git a/apply/processor/delete.go b/apply/processor/delete.go
@@ -17,20 +17,18 @@ package processor
 import (
 	"fmt"
 
-	common2 "github.com/sealerio/sealer/pkg/define/options"
-
-	"github.com/sealerio/sealer/pkg/imageengine"
-	"github.com/sealerio/sealer/pkg/registry"
-
 	"github.com/sealerio/sealer/common"
 	"github.com/sealerio/sealer/pkg/clusterfile"
+	common2 "github.com/sealerio/sealer/pkg/define/options"
 	"github.com/sealerio/sealer/pkg/filesystem"
 	"github.com/sealerio/sealer/pkg/filesystem/cloudfilesystem"
 	"github.com/sealerio/sealer/pkg/filesystem/clusterimage"
+	"github.com/sealerio/sealer/pkg/imageengine"
 	"github.com/sealerio/sealer/pkg/plugin"
-	"github.com/sealerio/sealer/pkg/runtime/kubernetes"
+	"github.com/sealerio/sealer/pkg/registry"
 	v2 "github.com/sealerio/sealer/types/api/v2"
 	utilsnet "github.com/sealerio/sealer/utils/net"
+	"github.com/sealerio/sealer/utils/platform"
 )
 
 type DeleteProcessor struct {
@@ -40,7 +38,7 @@ type DeleteProcessor struct {
 }
 
 func (d *DeleteProcessor) Reset(cluster *v2.Cluster) error {
-	runTime, err := kubernetes.NewDefaultRuntime(cluster, d.ClusterFile.GetKubeadmConfig())
+	runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, d.ClusterFile.GetKubeadmConfig())
 	if err != nil {
 		return fmt.Errorf("failed to init runtime: %v", err)
 	}

diff --git a/apply/processor/gen.go b/apply/processor/gen.go
@@ -19,22 +19,17 @@ package processor
 import (
 	"fmt"
 	"net"
-
-	"github.com/sealerio/sealer/pkg/auth"
-
-	"github.com/sealerio/sealer/pkg/define/options"
-
 	"strconv"
 
-	"github.com/sealerio/sealer/pkg/imageengine"
-
-	"github.com/sealerio/sealer/pkg/registry"
-
 	"github.com/sealerio/sealer/common"
+	"github.com/sealerio/sealer/pkg/auth"
 	"github.com/sealerio/sealer/pkg/client/k8s"
 	"github.com/sealerio/sealer/pkg/clusterfile"
+	"github.com/sealerio/sealer/pkg/define/options"
 	"github.com/sealerio/sealer/pkg/filesystem"
 	"github.com/sealerio/sealer/pkg/filesystem/clusterimage"
+	"github.com/sealerio/sealer/pkg/imageengine"
+	"github.com/sealerio/sealer/pkg/registry"
 	"github.com/sealerio/sealer/pkg/runtime/kubernetes"
 	v2 "github.com/sealerio/sealer/types/api/v2"
 	utilsnet "github.com/sealerio/sealer/utils/net"
@@ -194,7 +189,7 @@ func (g *GenerateProcessor) MountImage(cluster *v2.Cluster) error {
 	if err = g.ImageMounter.MountImage(cluster); err != nil {
 		return err
 	}
-	runt, err := kubernetes.NewDefaultRuntime(cluster, nil)
+	runt, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil)
 	if err != nil {
 		return err
 	}
@@ -207,7 +202,7 @@ func (g *GenerateProcessor) UnmountImage(cluster *v2.Cluster) error {
 }
 
 func (g *GenerateProcessor) ApplyRegistry(cluster *v2.Cluster) error {
-	runt, err := kubernetes.NewDefaultRuntime(cluster, nil)
+	runt, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil)
 	if err != nil {
 		return err
 	}

diff --git a/apply/processor/scale.go b/apply/processor/scale.go
@@ -18,19 +18,16 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/sealerio/sealer/utils/platform"
-
-	"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"
-
 	"github.com/sealerio/sealer/common"
 	"github.com/sealerio/sealer/pkg/clusterfile"
 	"github.com/sealerio/sealer/pkg/config"
 	"github.com/sealerio/sealer/pkg/filesystem"
 	"github.com/sealerio/sealer/pkg/filesystem/cloudfilesystem"
 	"github.com/sealerio/sealer/pkg/plugin"
 	"github.com/sealerio/sealer/pkg/runtime"
-	"github.com/sealerio/sealer/pkg/runtime/kubernetes"
+	"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"
 	v2 "github.com/sealerio/sealer/types/api/v2"
+	platform "github.com/sealerio/sealer/utils/platform"
 )
 
 type ScaleProcessor struct {
@@ -74,14 +71,9 @@ func (s *ScaleProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error
 }
 
 func (s *ScaleProcessor) PreProcess(cluster *v2.Cluster) error {
-	runTime, err := kubernetes.NewDefaultRuntime(cluster, s.KubeadmConfig)
-	if err != nil {
-		return fmt.Errorf("failed to init default runtime: %v", err)
-	}
-	s.Runtime = runTime
 	s.Config = config.NewConfiguration(platform.DefaultMountClusterImageDir(cluster.Name))
 	if s.IsScaleUp {
-		if err = clusterfile.SaveToDisk(cluster, cluster.Name); err != nil {
+		if err := clusterfile.SaveToDisk(cluster, cluster.Name); err != nil {
 			return err
 		}
 	}
@@ -130,7 +122,8 @@ func (s *ScaleProcessor) Delete(cluster *v2.Cluster) error {
 }
 
 func NewScaleProcessor(kubeadmConfig *kubeadm.KubeadmConfig, clusterFile clusterfile.Interface, masterToJoin, masterToDelete, nodeToJoin, nodeToDelete []net.IP) (Processor, error) {
-	fs, err := filesystem.NewFilesystem(common.DefaultTheClusterRootfsDir(clusterFile.GetCluster().Name))
+	cluster := clusterFile.GetCluster()
+	fs, err := filesystem.NewFilesystem(common.DefaultTheClusterRootfsDir(cluster.Name))
 	if err != nil {
 		return nil, err
 	}
@@ -141,7 +134,13 @@ func NewScaleProcessor(kubeadmConfig *kubeadm.KubeadmConfig, clusterFile cluster
 		up = true
 	}
 
+	runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), &cluster, clusterFile.GetKubeadmConfig())
+	if err != nil {
+		return nil, fmt.Errorf("failed to init default runtime: %v", err)
+	}
+
 	return &ScaleProcessor{
+		Runtime:         runTime,
 		MastersToDelete: masterToDelete,
 		MastersToJoin:   masterToJoin,
 		NodesToDelete:   nodeToDelete,

diff --git a/apply/processor/utils.go b/apply/processor/utils.go
@@ -0,0 +1,53 @@
+// Copyright © 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package processor
+
+import (
+	"github.com/sealerio/sealer/pkg/runtime"
+	"github.com/sealerio/sealer/pkg/runtime/k0s"
+	"github.com/sealerio/sealer/pkg/runtime/kubernetes"
+	"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"
+	v2 "github.com/sealerio/sealer/types/api/v2"
+	"github.com/sirupsen/logrus"
+)
+
+func ChooseRuntime(rootfs string, cluster *v2.Cluster, config *kubeadm.KubeadmConfig) (runtime.Interface, error) {
+	metadata, err := runtime.LoadMetadata(rootfs)
+	if err != nil {
+		return nil, err
+	}
+
+	hintInfo(metadata.ClusterRuntime)
+
+	switch metadata.ClusterRuntime {
+	case runtime.K8s:
+		return kubernetes.NewDefaultRuntime(cluster, config)
+	case runtime.K0s:
+		return k0s.NewK0sRuntime(cluster)
+	// Todo case runtime.K3s:
+	default:
+		return kubernetes.NewDefaultRuntime(cluster, config)
+	}
+}
+
+// hintInfo will hint end-user which cluster runtime will load and move to using new version ClusterImage.
+func hintInfo(clusterRuntime runtime.ClusterRuntime) {
+	if clusterRuntime != "" {
+		logrus.Infof("using %s as a cluster runtime.", clusterRuntime)
+		return
+	}
+	logrus.Infof("using k8s as a default cluster runtime.")
+	logrus.Infof("the old sealer version is deprecated, using new version of sealer and ClusterImage is encouraged!")
+}
diff --git a/cmd/sealer/cmd/cluster/apply.go b/cmd/sealer/cmd/cluster/apply.go
@@ -15,10 +15,10 @@
 package cluster
 
 import (
-	"github.com/sealerio/sealer/pkg/runtime/kubernetes"
-	"github.com/spf13/cobra"
-
 	"github.com/sealerio/sealer/apply"
+	"github.com/sealerio/sealer/pkg/runtime"
+
+	"github.com/spf13/cobra"
 )
 
 var clusterFile string
@@ -44,6 +44,6 @@ func NewApplyCmd() *cobra.Command {
 		},
 	}
 	applyCmd.Flags().StringVarP(&clusterFile, "Clusterfile", "f", "Clusterfile", "Clusterfile path to apply a Kubernetes cluster")
-	applyCmd.Flags().BoolVar(&kubernetes.ForceDelete, "force", false, "force to delete the specified cluster if set true")
+	applyCmd.Flags().BoolVar(&runtime.ForceDelete, "force", false, "force to delete the specified cluster if set true")
 	return applyCmd
 }
diff --git a/cmd/sealer/cmd/cluster/delete.go b/cmd/sealer/cmd/cluster/delete.go
@@ -19,9 +19,8 @@ import (
 
 	"github.com/sealerio/sealer/apply"
 	"github.com/sealerio/sealer/common"
-
 	"github.com/sealerio/sealer/pkg/clusterfile"
-	"github.com/sealerio/sealer/pkg/runtime/kubernetes"
+	"github.com/sealerio/sealer/pkg/runtime"
 
 	"github.com/spf13/cobra"
 )
@@ -100,7 +99,7 @@ func NewDeleteCmd() *cobra.Command {
 	deleteCmd.Flags().StringVarP(&deleteClusterFile, "Clusterfile", "f", "", "delete a kubernetes cluster with Clusterfile Annotations")
 	deleteCmd.Flags().StringVarP(&deleteClusterName, "cluster", "c", "", "delete a kubernetes cluster with cluster name")
 	deleteCmd.Flags().StringSliceVarP(&deleteArgs.CustomEnv, "env", "e", []string{}, "set custom environment variables")
-	deleteCmd.Flags().BoolVar(&kubernetes.ForceDelete, "force", false, "We also can input an --force flag to delete cluster by force")
+	deleteCmd.Flags().BoolVar(&runtime.ForceDelete, "force", false, "We also can input an --force flag to delete cluster by force")
 	deleteCmd.Flags().BoolP("all", "a", false, "this flags is for delete nodes, if this is true, empty all node ip")
 	return deleteCmd
 }
diff --git a/pkg/runtime/interface.go b/pkg/runtime/interface.go
@@ -36,3 +36,5 @@ type Interface interface {
 	// GetClusterMetadata read the rootfs/Metadata file to get some install info for cluster.
 	GetClusterMetadata() (*Metadata, error)
 }
+
+var ForceDelete bool
diff --git a/pkg/runtime/k0s/common.go b/pkg/runtime/k0s/common.go
@@ -15,11 +15,16 @@
 package k0s
 
 const (
+	RuntimeFlag = "k0s"
+	VersionCmd  = "k0s version"
+
 	DefaultAdminConf = "/var/lib/k0s/pki/admin.conf"
 
 	DefaultK0sConfigPath     = "/etc/k0s/k0s.yaml"
 	DefaultK0sWorkerJoin     = "/etc/k0s/worker"
 	DefaultK0sControllerJoin = "/etc/k0s/controller"
 	WorkerRole               = "worker"
 	ControllerRole           = "controller"
+
+	ExternalCRI = "/run/containerd/containerd.sock"
 )