feat: add k0s runtime join/delete/reset/upgrade.

Signed-off-by: starComingup <[email protected]>

pkg/runtime/k0s/common.go

@@ -15,11 +15,16 @@
 package k0s
 
 const (
+	RuntimeFlag = "k0s"
+	VersionCmd  = "k0s version"
+
 	DefaultAdminConf = "/var/lib/k0s/pki/admin.conf"
 
 	DefaultK0sConfigPath     = "/etc/k0s/k0s.yaml"
 	DefaultK0sWorkerJoin     = "/etc/k0s/worker"
 	DefaultK0sControllerJoin = "/etc/k0s/controller"
 	WorkerRole               = "worker"
 	ControllerRole           = "controller"
+
+	ExternalCRI = "/run/containerd/containerd.sock"
 )

pkg/runtime/k0s/delete_masters.go

@@ -0,0 +1,124 @@
+// Copyright © 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k0s
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	RemoteCleanMasterOrNode = `if which k0s; then k0s stop && k0s reset --config %s --cri-socket %s;fi && \
+rm -rf /etc/k0s/
+`
+	RemoveKubeConfig          = "rm -rf /usr/bin/kube* && rm -rf ~/.kube/"
+	RemoveK0sBin              = "rm -rf /usr/bin/k0s"
+	RemoteRemoveEtcHost       = "sed -i \"/%s/d\" /etc/hosts"
+	RemoteRemoveRegistryCerts = "rm -rf " + DockerCertDir + "/%s*"
+	KubeDeleteNode            = "kubectl delete node %s"
+)
+
+func (k *Runtime) deleteMasters(masters []net.IP) error {
+	if len(masters) == 0 {
+		return nil
+	}
+	eg, _ := errgroup.WithContext(context.Background())
+	for _, master := range masters {
+		master := master
+		eg.Go(func() error {
+			master := master
+			logrus.Infof("Start to delete master %s", master)
+			if err := k.deleteMaster(master); err != nil {
+				logrus.Errorf("failed to delete master %s: %v", master, err)
+			} else {
+				logrus.Infof("Succeeded in deleting master %s", master)
+			}
+			return nil
+		})
+	}
+	return eg.Wait()
+}
+
+func (k *Runtime) deleteMaster(master net.IP) error {
+	ssh, err := k.getHostSSHClient(master)
+	if err != nil {
+		return fmt.Errorf("failed to delete master: %v", err)
+	}
+	remoteCleanCmd := []string{fmt.Sprintf(RemoteCleanMasterOrNode, DefaultK0sConfigPath, ExternalCRI),
+		fmt.Sprintf(RemoteRemoveEtcHost, SeaHub),
+		fmt.Sprintf(RemoteRemoveRegistryCerts, k.RegConfig.Domain),
+		fmt.Sprintf(RemoteRemoveRegistryCerts, SeaHub),
+		RemoveKubeConfig,
+		RemoveK0sBin}
+
+	if err := ssh.CmdAsync(master, remoteCleanCmd...); err != nil {
+		return err
+	}
+
+	// remove master
+	masterIPs := []net.IP{}
+	for _, ip := range k.cluster.GetMasterIPList() {
+		if !ip.Equal(master) {
+			masterIPs = append(masterIPs, ip)
+		}
+	}
+
+	if len(masterIPs) > 0 {
+		hostname, err := k.isHostName(k.cluster.GetMaster0IP(), master)
+		if err != nil {
+			return err
+		}
+		master0SSH, err := k.getHostSSHClient(k.cluster.GetMaster0IP())
+		if err != nil {
+			return fmt.Errorf("failed to get master0 ssh client: %v", err)
+		}
+
+		if err := master0SSH.CmdAsync(k.cluster.GetMaster0IP(), fmt.Sprintf(KubeDeleteNode, strings.TrimSpace(hostname))); err != nil {
+			return fmt.Errorf("failed to delete node %s: %v", hostname, err)
+		}
+	}
+	return nil
+}
+
+func (k *Runtime) isHostName(master, host net.IP) (string, error) {
+	hostString, err := k.CmdToString(master, "kubectl get nodes | grep -v NAME  | awk '{print $1}'", ",")
+	if err != nil {
+		return "", err
+	}
+	hostName, err := k.CmdToString(host, "hostname", "")
+	if err != nil {
+		return "", err
+	}
+	hosts := strings.Split(hostString, ",")
+	var name string
+	for _, h := range hosts {
+		if strings.TrimSpace(h) == "" {
+			continue
+		} else {
+			hh := strings.ToLower(h)
+			fromH := strings.ToLower(hostName)
+			if hh == fromH {
+				name = h
+				break
+			}
+		}
+	}
+	return name, nil
+}

pkg/runtime/k0s/delete_nodes.go

@@ -0,0 +1,78 @@
+// Copyright © 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k0s
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+)
+
+func (k *Runtime) deleteNodes(nodes []net.IP) error {
+	if len(nodes) == 0 {
+		return nil
+	}
+	eg, _ := errgroup.WithContext(context.Background())
+	for _, node := range nodes {
+		node := node
+		eg.Go(func() error {
+			logrus.Infof("Start to delete worker %s", node)
+			if err := k.deleteNode(node); err != nil {
+				return fmt.Errorf("failed to delete node %s: %v", node, err)
+			}
+			logrus.Infof("Succeeded in deleting worker %s", node)
+			return nil
+		})
+	}
+	return eg.Wait()
+}
+
+func (k *Runtime) deleteNode(node net.IP) error {
+	ssh, err := k.getHostSSHClient(node)
+	if err != nil {
+		return fmt.Errorf("failed to delete node: %v", err)
+	}
+	remoteCleanCmds := []string{fmt.Sprintf(RemoteCleanMasterOrNode, DefaultK0sConfigPath, ExternalCRI),
+		fmt.Sprintf(RemoteRemoveEtcHost, k.RegConfig.Domain),
+		fmt.Sprintf(RemoteRemoveEtcHost, SeaHub),
+		fmt.Sprintf(RemoteRemoveRegistryCerts, k.RegConfig.Domain),
+		fmt.Sprintf(RemoteRemoveRegistryCerts, SeaHub),
+		RemoveKubeConfig,
+		RemoveK0sBin}
+	if err := ssh.CmdAsync(node, remoteCleanCmds...); err != nil {
+		return err
+	}
+
+	//remove node
+	if len(k.cluster.GetMasterIPList()) > 0 {
+		hostname, err := k.isHostName(k.cluster.GetMaster0IP(), node)
+		if err != nil {
+			return err
+		}
+		ssh, err := k.getHostSSHClient(k.cluster.GetMaster0IP())
+		if err != nil {
+			return fmt.Errorf("failed to get master0 ssh client(%s): %v", k.cluster.GetMaster0IP(), err)
+		}
+		if err := ssh.CmdAsync(k.cluster.GetMaster0IP(), fmt.Sprintf(KubeDeleteNode, strings.TrimSpace(hostname))); err != nil {
+			return fmt.Errorf("failed to delete node %s: %v", hostname, err)
+		}
+	}
+
+	return nil
+}

pkg/runtime/k0s/join_masters.go

@@ -0,0 +1,82 @@
+// Copyright © 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k0s
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/sealerio/sealer/common"
+	"github.com/sealerio/sealer/utils/ssh"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const RemoteNonRootCopyKubeConfig = `rm -rf ${HOME}/.kube/config && mkdir -p ${HOME}/.kube && cp /var/lib/k0s/pki/admin.conf ${HOME}/.kube/config && chown $(id -u):$(id -g) ${HOME}/.kube/config`
+
+func (k *Runtime) joinMasters(masters []net.IP) error {
+	if len(masters) == 0 {
+		return nil
+	}
+	if err := k.WaitSSHReady(6, masters...); err != nil {
+		return errors.Wrap(err, "join masters wait for ssh ready time out")
+	}
+	if err := k.CopyJoinToken(ControllerRole, masters); err != nil {
+		return err
+	}
+	if err := k.sendRegistryCert(masters); err != nil {
+		return err
+	}
+	cmds := k.Command(ControllerRole)
+	if cmds == nil {
+		return fmt.Errorf("failed to get join master command")
+	}
+
+	for _, master := range masters {
+		logrus.Infof("Start to join %s as master", master)
+
+		masterCmds := k.JoinMasterCommands(cmds)
+		client, err := k.getHostSSHClient(master)
+		if err != nil {
+			return err
+		}
+
+		if client.(*ssh.SSH).User != common.ROOT {
+			masterCmds = append(masterCmds, RemoteNonRootCopyKubeConfig)
+		}
+
+		if err := client.CmdAsync(master, masterCmds...); err != nil {
+			return fmt.Errorf("failed to exec command(%s) on master(%s): %v", cmds, master, err)
+		}
+
+		logrus.Infof("Succeeded in joining %s as master", master)
+	}
+	return nil
+}
+
+func (k *Runtime) JoinMasterCommands(cmds []string) []string {
+	cmdAddRegistryHosts := k.addRegistryDomainToHosts()
+	if k.RegConfig.Domain != SeaHub {
+		cmdAddSeaHubHosts := fmt.Sprintf(RemoteAddEtcHosts, k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub)
+		cmdAddRegistryHosts = fmt.Sprintf("%s && %s", cmdAddRegistryHosts, cmdAddSeaHubHosts)
+	}
+	joinCommands := []string{cmdAddRegistryHosts}
+	if k.RegConfig.Username != "" && k.RegConfig.Password != "" {
+		joinCommands = append(joinCommands, k.GenLoginCommand())
+	}
+
+	return append(joinCommands, cmds...)
+}

pkg/runtime/k0s/join_nodes.go

@@ -0,0 +1,72 @@
+// Copyright © 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k0s
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+)
+
+func (k *Runtime) joinNodes(nodes []net.IP) error {
+	if len(nodes) == 0 {
+		return nil
+	}
+	if err := k.WaitSSHReady(6, nodes...); err != nil {
+		return errors.Wrap(err, "join nodes wait for ssh ready time out")
+	}
+	if err := k.sendRegistryCert(nodes); err != nil {
+		return err
+	}
+	if err := k.CopyJoinToken(WorkerRole, nodes); err != nil {
+		return err
+	}
+	addRegistryHostsAndLogin := k.addRegistryDomainToHosts()
+	if k.RegConfig.Domain != SeaHub {
+		addSeaHubHost := fmt.Sprintf(RemoteAddEtcHosts, k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub)
+		addRegistryHostsAndLogin = fmt.Sprintf("%s && %s", addRegistryHostsAndLogin, addSeaHubHost)
+	}
+	if k.RegConfig.Username != "" && k.RegConfig.Password != "" {
+		addRegistryHostsAndLogin = fmt.Sprintf("%s && %s", addRegistryHostsAndLogin, k.GenLoginCommand())
+	}
+	cmds := k.Command(WorkerRole)
+	if cmds == nil {
+		return fmt.Errorf("failed to get join node command")
+	}
+
+	eg, _ := errgroup.WithContext(context.Background())
+	for _, node := range nodes {
+		node := node
+		eg.Go(func() error {
+			logrus.Infof("Start to join %s as worker", node)
+
+			nodeCmds := append([]string{addRegistryHostsAndLogin}, cmds...)
+			ssh, err := k.getHostSSHClient(node)
+			if err != nil {
+				return fmt.Errorf("failed to join node %s: %v", node, err)
+			}
+			if err := ssh.CmdAsync(node, nodeCmds...); err != nil {
+				return fmt.Errorf("failed to join node %s: %v", node, err)
+			}
+			logrus.Infof("Succeeded in joining %s as worker", node)
+			return err
+		})
+	}
+	return eg.Wait()
+}

pkg/runtime/k0s/registry.go

@@ -26,6 +26,8 @@ const (
 	ContainerdLoginCommand      = "nerdctl login -u %s -p %s %s"
 	DefaultRegistryHtPasswdFile = "registry_htpasswd"
 	DockerCertDir               = "/etc/docker/certs.d"
+	DeleteRegistryCommand       = "((! nerdctl ps -a 2>/dev/null |grep %[1]s) || (nerdctl stop %[1]s && nerdctl rmi -f %[1]s))"
+	RegistryName                = "sealer-registry"
 )
 
 // sendRegistryCertAndKey send registry cert to Master0 host. path like: /var/lib/sealer/data/my-k0s-cluster/certs
@@ -100,3 +102,12 @@ func (k *Runtime) SendRegistryCert(host []net.IP) error {
 	}
 	return k.sendRegistryCert(host)
 }
+
+func (k *Runtime) DeleteRegistry() error {
+	ssh, err := k.getHostSSHClient(k.RegConfig.IP)
+	if err != nil {
+		return fmt.Errorf("failed to delete registry: %v", err)
+	}
+
+	return ssh.CmdAsync(k.RegConfig.IP, fmt.Sprintf(DeleteRegistryCommand, RegistryName))
+}