Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: k0s runtime remain work #1686

Merged
merged 5 commits into from
Sep 15, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 4 additions & 7 deletions apply/driver/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,15 @@ import (
"fmt"
"net"

imagecommon "github.com/sealerio/sealer/pkg/define/options"

"github.com/sealerio/sealer/pkg/auth"
"github.com/sealerio/sealer/pkg/imageengine"

"github.com/sealerio/sealer/apply/processor"
"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/auth"
"github.com/sealerio/sealer/pkg/client/k8s"
"github.com/sealerio/sealer/pkg/clusterfile"
imagecommon "github.com/sealerio/sealer/pkg/define/options"
"github.com/sealerio/sealer/pkg/filesystem/clusterimage"
"github.com/sealerio/sealer/pkg/imageengine"
"github.com/sealerio/sealer/pkg/runtime"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
v2 "github.com/sealerio/sealer/types/api/v2"
"github.com/sealerio/sealer/utils"
osi "github.com/sealerio/sealer/utils/os"
Expand Down Expand Up @@ -237,7 +234,7 @@ func (applier *Applier) Upgrade(upgradeImgName string) error {
}

func (applier *Applier) upgrade() error {
runtimeInterface, err := kubernetes.NewDefaultRuntime(applier.ClusterDesired, applier.ClusterFile.GetKubeadmConfig())
runtimeInterface, err := processor.ChooseRuntime(platform.DefaultMountClusterImageDir(applier.ClusterDesired.Name), applier.ClusterDesired, applier.ClusterFile.GetKubeadmConfig())
if err != nil {
return fmt.Errorf("failed to init runtime: %v", err)
}
Expand Down
4 changes: 2 additions & 2 deletions apply/processor/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"github.com/sealerio/sealer/pkg/guest"
"github.com/sealerio/sealer/pkg/plugin"
"github.com/sealerio/sealer/pkg/runtime"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
v2 "github.com/sealerio/sealer/types/api/v2"
"github.com/sealerio/sealer/utils/net"
"github.com/sealerio/sealer/utils/platform"
Expand Down Expand Up @@ -111,7 +110,8 @@ func (c *CreateProcessor) MountImage(cluster *v2.Cluster) error {
if err = c.cloudImageMounter.MountImage(cluster); err != nil {
return err
}
runTime, err := kubernetes.NewDefaultRuntime(cluster, c.ClusterFile.GetKubeadmConfig())
//TODO split kubeadm config from cluster file.
runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, c.ClusterFile.GetKubeadmConfig())
if err != nil {
return fmt.Errorf("failed to init runtime: %v", err)
}
Expand Down
12 changes: 5 additions & 7 deletions apply/processor/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,18 @@ package processor
import (
"fmt"

common2 "github.com/sealerio/sealer/pkg/define/options"

"github.com/sealerio/sealer/pkg/imageengine"
"github.com/sealerio/sealer/pkg/registry"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/clusterfile"
common2 "github.com/sealerio/sealer/pkg/define/options"
"github.com/sealerio/sealer/pkg/filesystem"
"github.com/sealerio/sealer/pkg/filesystem/cloudfilesystem"
"github.com/sealerio/sealer/pkg/filesystem/clusterimage"
"github.com/sealerio/sealer/pkg/imageengine"
"github.com/sealerio/sealer/pkg/plugin"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
"github.com/sealerio/sealer/pkg/registry"
v2 "github.com/sealerio/sealer/types/api/v2"
utilsnet "github.com/sealerio/sealer/utils/net"
"github.com/sealerio/sealer/utils/platform"
)

type DeleteProcessor struct {
Expand All @@ -40,7 +38,7 @@ type DeleteProcessor struct {
}

func (d *DeleteProcessor) Reset(cluster *v2.Cluster) error {
runTime, err := kubernetes.NewDefaultRuntime(cluster, d.ClusterFile.GetKubeadmConfig())
runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, d.ClusterFile.GetKubeadmConfig())
if err != nil {
return fmt.Errorf("failed to init runtime: %v", err)
}
Expand Down
17 changes: 6 additions & 11 deletions apply/processor/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,17 @@ package processor
import (
"fmt"
"net"

"github.com/sealerio/sealer/pkg/auth"

"github.com/sealerio/sealer/pkg/define/options"

"strconv"

"github.com/sealerio/sealer/pkg/imageengine"

"github.com/sealerio/sealer/pkg/registry"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/auth"
"github.com/sealerio/sealer/pkg/client/k8s"
"github.com/sealerio/sealer/pkg/clusterfile"
"github.com/sealerio/sealer/pkg/define/options"
"github.com/sealerio/sealer/pkg/filesystem"
"github.com/sealerio/sealer/pkg/filesystem/clusterimage"
"github.com/sealerio/sealer/pkg/imageengine"
"github.com/sealerio/sealer/pkg/registry"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
v2 "github.com/sealerio/sealer/types/api/v2"
utilsnet "github.com/sealerio/sealer/utils/net"
Expand Down Expand Up @@ -194,7 +189,7 @@ func (g *GenerateProcessor) MountImage(cluster *v2.Cluster) error {
if err = g.ImageMounter.MountImage(cluster); err != nil {
return err
}
runt, err := kubernetes.NewDefaultRuntime(cluster, nil)
runt, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil)
if err != nil {
return err
}
Expand All @@ -207,7 +202,7 @@ func (g *GenerateProcessor) UnmountImage(cluster *v2.Cluster) error {
}

func (g *GenerateProcessor) ApplyRegistry(cluster *v2.Cluster) error {
runt, err := kubernetes.NewDefaultRuntime(cluster, nil)
runt, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil)
if err != nil {
return err
}
Expand Down
23 changes: 11 additions & 12 deletions apply/processor/scale.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,16 @@ import (
"fmt"
"net"

"github.com/sealerio/sealer/utils/platform"

"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/clusterfile"
"github.com/sealerio/sealer/pkg/config"
"github.com/sealerio/sealer/pkg/filesystem"
"github.com/sealerio/sealer/pkg/filesystem/cloudfilesystem"
"github.com/sealerio/sealer/pkg/plugin"
"github.com/sealerio/sealer/pkg/runtime"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"
v2 "github.com/sealerio/sealer/types/api/v2"
platform "github.com/sealerio/sealer/utils/platform"
)

type ScaleProcessor struct {
Expand Down Expand Up @@ -74,14 +71,9 @@ func (s *ScaleProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error
}

func (s *ScaleProcessor) PreProcess(cluster *v2.Cluster) error {
runTime, err := kubernetes.NewDefaultRuntime(cluster, s.KubeadmConfig)
if err != nil {
return fmt.Errorf("failed to init default runtime: %v", err)
}
s.Runtime = runTime
s.Config = config.NewConfiguration(platform.DefaultMountClusterImageDir(cluster.Name))
if s.IsScaleUp {
if err = clusterfile.SaveToDisk(cluster, cluster.Name); err != nil {
if err := clusterfile.SaveToDisk(cluster, cluster.Name); err != nil {
return err
}
}
Expand Down Expand Up @@ -130,7 +122,8 @@ func (s *ScaleProcessor) Delete(cluster *v2.Cluster) error {
}

func NewScaleProcessor(kubeadmConfig *kubeadm.KubeadmConfig, clusterFile clusterfile.Interface, masterToJoin, masterToDelete, nodeToJoin, nodeToDelete []net.IP) (Processor, error) {
fs, err := filesystem.NewFilesystem(common.DefaultTheClusterRootfsDir(clusterFile.GetCluster().Name))
cluster := clusterFile.GetCluster()
fs, err := filesystem.NewFilesystem(common.DefaultTheClusterRootfsDir(cluster.Name))
if err != nil {
return nil, err
}
Expand All @@ -141,7 +134,13 @@ func NewScaleProcessor(kubeadmConfig *kubeadm.KubeadmConfig, clusterFile cluster
up = true
}

runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), &cluster, clusterFile.GetKubeadmConfig())
if err != nil {
return nil, fmt.Errorf("failed to init default runtime: %v", err)
}

return &ScaleProcessor{
Runtime: runTime,
MastersToDelete: masterToDelete,
MastersToJoin: masterToJoin,
NodesToDelete: nodeToDelete,
Expand Down
53 changes: 53 additions & 0 deletions apply/processor/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright © 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package processor

import (
"github.com/sealerio/sealer/pkg/runtime"
"github.com/sealerio/sealer/pkg/runtime/k0s"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"
v2 "github.com/sealerio/sealer/types/api/v2"
"github.com/sirupsen/logrus"
)

func ChooseRuntime(rootfs string, cluster *v2.Cluster, config *kubeadm.KubeadmConfig) (runtime.Interface, error) {
metadata, err := runtime.LoadMetadata(rootfs)
if err != nil {
return nil, err
}

hintInfo(metadata.ClusterRuntime)

switch metadata.ClusterRuntime {
case runtime.K8s:
return kubernetes.NewDefaultRuntime(cluster, config)
case runtime.K0s:
return k0s.NewK0sRuntime(cluster)
// Todo case runtime.K3s:
default:
return kubernetes.NewDefaultRuntime(cluster, config)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wish to see more thinking details about the default case. From my point of view, if there is no metadata.ClusterRuntime in image rootfs details, we take it as k8s by default. Because in the original implementation of ClusterImage, there is no metadata.ClusterRuntime, and we have to make the new version of sealer compatible with the legacy ClusterImage format. For what is worth, I suggest to add a log to tell end-user that he is using a legacy version, and using new version of sealer and ClusterImage is encouraged.

Otherwise, I think you should add a line of explicit log at Line31 to show the ClusterRuntime info.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with add a cluster runtime info and hint end-user using new version ClusterImage.

}
}

// hintInfo will hint end-user which cluster runtime will load and move to using new version ClusterImage.
func hintInfo(clusterRuntime runtime.ClusterRuntime) {
if clusterRuntime != "" {
logrus.Infof("using %s as a cluster runtime.", clusterRuntime)
return
}
logrus.Infof("using k8s as a default cluster runtime.")
logrus.Infof("the old sealer version is deprecated, using new version of sealer and ClusterImage is encouraged!")
}
8 changes: 4 additions & 4 deletions cmd/sealer/cmd/cluster/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
package cluster

import (
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
"github.com/spf13/cobra"

"github.com/sealerio/sealer/apply"
"github.com/sealerio/sealer/pkg/runtime"

"github.com/spf13/cobra"
)

var clusterFile string
Expand All @@ -44,6 +44,6 @@ func NewApplyCmd() *cobra.Command {
},
}
applyCmd.Flags().StringVarP(&clusterFile, "Clusterfile", "f", "Clusterfile", "Clusterfile path to apply a Kubernetes cluster")
applyCmd.Flags().BoolVar(&kubernetes.ForceDelete, "force", false, "force to delete the specified cluster if set true")
applyCmd.Flags().BoolVar(&runtime.ForceDelete, "force", false, "force to delete the specified cluster if set true")
return applyCmd
}
5 changes: 2 additions & 3 deletions cmd/sealer/cmd/cluster/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ import (

"github.com/sealerio/sealer/apply"
"github.com/sealerio/sealer/common"

"github.com/sealerio/sealer/pkg/clusterfile"
"github.com/sealerio/sealer/pkg/runtime/kubernetes"
"github.com/sealerio/sealer/pkg/runtime"

"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -100,7 +99,7 @@ func NewDeleteCmd() *cobra.Command {
deleteCmd.Flags().StringVarP(&deleteClusterFile, "Clusterfile", "f", "", "delete a kubernetes cluster with Clusterfile Annotations")
deleteCmd.Flags().StringVarP(&deleteClusterName, "cluster", "c", "", "delete a kubernetes cluster with cluster name")
deleteCmd.Flags().StringSliceVarP(&deleteArgs.CustomEnv, "env", "e", []string{}, "set custom environment variables")
deleteCmd.Flags().BoolVar(&kubernetes.ForceDelete, "force", false, "We also can input an --force flag to delete cluster by force")
deleteCmd.Flags().BoolVar(&runtime.ForceDelete, "force", false, "We also can input an --force flag to delete cluster by force")
deleteCmd.Flags().BoolP("all", "a", false, "this flags is for delete nodes, if this is true, empty all node ip")
return deleteCmd
}
2 changes: 2 additions & 0 deletions pkg/runtime/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,5 @@ type Interface interface {
// GetClusterMetadata read the rootfs/Metadata file to get some install info for cluster.
GetClusterMetadata() (*Metadata, error)
}

var ForceDelete bool
5 changes: 5 additions & 0 deletions pkg/runtime/k0s/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,16 @@
package k0s

const (
RuntimeFlag = "k0s"
VersionCmd = "k0s version"

DefaultAdminConf = "/var/lib/k0s/pki/admin.conf"

DefaultK0sConfigPath = "/etc/k0s/k0s.yaml"
DefaultK0sWorkerJoin = "/etc/k0s/worker"
DefaultK0sControllerJoin = "/etc/k0s/controller"
WorkerRole = "worker"
ControllerRole = "controller"

ExternalCRI = "/run/containerd/containerd.sock"
)
Loading