From 053e716971113b29edb7594f30dd73fba97b4239 Mon Sep 17 00:00:00 2001 From: starComingup <1225067236@qq.com> Date: Mon, 5 Sep 2022 15:42:05 +0800 Subject: [PATCH 1/5] feat: add k0s runtime join/delete/reset/upgrade. Signed-off-by: starComingup <1225067236@qq.com> --- pkg/runtime/k0s/common.go | 5 + pkg/runtime/k0s/delete_masters.go | 124 ++++++++++++++++++++++ pkg/runtime/k0s/delete_nodes.go | 78 ++++++++++++++ pkg/runtime/k0s/join_masters.go | 82 +++++++++++++++ pkg/runtime/k0s/join_nodes.go | 72 +++++++++++++ pkg/runtime/k0s/registry.go | 11 ++ pkg/runtime/k0s/reset.go | 71 +++++++++++++ pkg/runtime/k0s/runtime.go | 165 ++++++++++++++++++++++++++---- pkg/runtime/k0s/upgrade.go | 97 ++++++++++++++++++ 9 files changed, 686 insertions(+), 19 deletions(-) create mode 100644 pkg/runtime/k0s/delete_masters.go create mode 100644 pkg/runtime/k0s/delete_nodes.go create mode 100644 pkg/runtime/k0s/join_masters.go create mode 100644 pkg/runtime/k0s/join_nodes.go create mode 100644 pkg/runtime/k0s/reset.go create mode 100644 pkg/runtime/k0s/upgrade.go diff --git a/pkg/runtime/k0s/common.go b/pkg/runtime/k0s/common.go index 96e59a4d7c8..8cb7203ac74 100644 --- a/pkg/runtime/k0s/common.go +++ b/pkg/runtime/k0s/common.go @@ -15,6 +15,9 @@ package k0s const ( + RuntimeFlag = "k0s" + VersionCmd = "k0s version" + DefaultAdminConf = "/var/lib/k0s/pki/admin.conf" DefaultK0sConfigPath = "/etc/k0s/k0s.yaml" @@ -22,4 +25,6 @@ const ( DefaultK0sControllerJoin = "/etc/k0s/controller" WorkerRole = "worker" ControllerRole = "controller" + + ExternalCRI = "/run/containerd/containerd.sock" ) diff --git a/pkg/runtime/k0s/delete_masters.go b/pkg/runtime/k0s/delete_masters.go new file mode 100644 index 00000000000..2d3136ec683 --- /dev/null +++ b/pkg/runtime/k0s/delete_masters.go @@ -0,0 +1,124 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k0s + +import ( + "context" + "fmt" + "net" + "strings" + + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +const ( + RemoteCleanMasterOrNode = `if which k0s; then k0s stop && k0s reset --config %s --cri-socket %s;fi && \ +rm -rf /etc/k0s/ +` + RemoveKubeConfig = "rm -rf /usr/bin/kube* && rm -rf ~/.kube/" + RemoveK0sBin = "rm -rf /usr/bin/k0s" + RemoteRemoveEtcHost = "sed -i \"/%s/d\" /etc/hosts" + RemoteRemoveRegistryCerts = "rm -rf " + DockerCertDir + "/%s*" + KubeDeleteNode = "kubectl delete node %s" +) + +func (k *Runtime) deleteMasters(masters []net.IP) error { + if len(masters) == 0 { + return nil + } + eg, _ := errgroup.WithContext(context.Background()) + for _, master := range masters { + master := master + eg.Go(func() error { + master := master + logrus.Infof("Start to delete master %s", master) + if err := k.deleteMaster(master); err != nil { + logrus.Errorf("failed to delete master %s: %v", master, err) + } else { + logrus.Infof("Succeeded in deleting master %s", master) + } + return nil + }) + } + return eg.Wait() +} + +func (k *Runtime) deleteMaster(master net.IP) error { + ssh, err := k.getHostSSHClient(master) + if err != nil { + return fmt.Errorf("failed to delete master: %v", err) + } + remoteCleanCmd := []string{fmt.Sprintf(RemoteCleanMasterOrNode, DefaultK0sConfigPath, ExternalCRI), + fmt.Sprintf(RemoteRemoveEtcHost, SeaHub), + fmt.Sprintf(RemoteRemoveRegistryCerts, k.RegConfig.Domain), + fmt.Sprintf(RemoteRemoveRegistryCerts, SeaHub), + RemoveKubeConfig, + RemoveK0sBin} + + if err := ssh.CmdAsync(master, remoteCleanCmd...); err != nil { + return err + } + + // remove master + masterIPs := []net.IP{} + for _, ip := range k.cluster.GetMasterIPList() { + if !ip.Equal(master) { + masterIPs = append(masterIPs, ip) + } + } + + if len(masterIPs) > 0 { + hostname, err := k.isHostName(k.cluster.GetMaster0IP(), master) + if err != nil { + return err + } + master0SSH, err := k.getHostSSHClient(k.cluster.GetMaster0IP()) + if err != nil { + return fmt.Errorf("failed to get master0 ssh client: %v", err) + } + + if err := master0SSH.CmdAsync(k.cluster.GetMaster0IP(), fmt.Sprintf(KubeDeleteNode, strings.TrimSpace(hostname))); err != nil { + return fmt.Errorf("failed to delete node %s: %v", hostname, err) + } + } + return nil +} + +func (k *Runtime) isHostName(master, host net.IP) (string, error) { + hostString, err := k.CmdToString(master, "kubectl get nodes | grep -v NAME | awk '{print $1}'", ",") + if err != nil { + return "", err + } + hostName, err := k.CmdToString(host, "hostname", "") + if err != nil { + return "", err + } + hosts := strings.Split(hostString, ",") + var name string + for _, h := range hosts { + if strings.TrimSpace(h) == "" { + continue + } else { + hh := strings.ToLower(h) + fromH := strings.ToLower(hostName) + if hh == fromH { + name = h + break + } + } + } + return name, nil +} diff --git a/pkg/runtime/k0s/delete_nodes.go b/pkg/runtime/k0s/delete_nodes.go new file mode 100644 index 00000000000..3d5436710d5 --- /dev/null +++ b/pkg/runtime/k0s/delete_nodes.go @@ -0,0 +1,78 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k0s + +import ( + "context" + "fmt" + "net" + "strings" + + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +func (k *Runtime) deleteNodes(nodes []net.IP) error { + if len(nodes) == 0 { + return nil + } + eg, _ := errgroup.WithContext(context.Background()) + for _, node := range nodes { + node := node + eg.Go(func() error { + logrus.Infof("Start to delete worker %s", node) + if err := k.deleteNode(node); err != nil { + return fmt.Errorf("failed to delete node %s: %v", node, err) + } + logrus.Infof("Succeeded in deleting worker %s", node) + return nil + }) + } + return eg.Wait() +} + +func (k *Runtime) deleteNode(node net.IP) error { + ssh, err := k.getHostSSHClient(node) + if err != nil { + return fmt.Errorf("failed to delete node: %v", err) + } + remoteCleanCmds := []string{fmt.Sprintf(RemoteCleanMasterOrNode, DefaultK0sConfigPath, ExternalCRI), + fmt.Sprintf(RemoteRemoveEtcHost, k.RegConfig.Domain), + fmt.Sprintf(RemoteRemoveEtcHost, SeaHub), + fmt.Sprintf(RemoteRemoveRegistryCerts, k.RegConfig.Domain), + fmt.Sprintf(RemoteRemoveRegistryCerts, SeaHub), + RemoveKubeConfig, + RemoveK0sBin} + if err := ssh.CmdAsync(node, remoteCleanCmds...); err != nil { + return err + } + + //remove node + if len(k.cluster.GetMasterIPList()) > 0 { + hostname, err := k.isHostName(k.cluster.GetMaster0IP(), node) + if err != nil { + return err + } + ssh, err := k.getHostSSHClient(k.cluster.GetMaster0IP()) + if err != nil { + return fmt.Errorf("failed to get master0 ssh client(%s): %v", k.cluster.GetMaster0IP(), err) + } + if err := ssh.CmdAsync(k.cluster.GetMaster0IP(), fmt.Sprintf(KubeDeleteNode, strings.TrimSpace(hostname))); err != nil { + return fmt.Errorf("failed to delete node %s: %v", hostname, err) + } + } + + return nil +} diff --git a/pkg/runtime/k0s/join_masters.go b/pkg/runtime/k0s/join_masters.go new file mode 100644 index 00000000000..9cae23d7e0e --- /dev/null +++ b/pkg/runtime/k0s/join_masters.go @@ -0,0 +1,82 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k0s + +import ( + "fmt" + "net" + + "github.com/sealerio/sealer/common" + "github.com/sealerio/sealer/utils/ssh" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const RemoteNonRootCopyKubeConfig = `rm -rf ${HOME}/.kube/config && mkdir -p ${HOME}/.kube && cp /var/lib/k0s/pki/admin.conf ${HOME}/.kube/config && chown $(id -u):$(id -g) ${HOME}/.kube/config` + +func (k *Runtime) joinMasters(masters []net.IP) error { + if len(masters) == 0 { + return nil + } + if err := k.WaitSSHReady(6, masters...); err != nil { + return errors.Wrap(err, "join masters wait for ssh ready time out") + } + if err := k.CopyJoinToken(ControllerRole, masters); err != nil { + return err + } + if err := k.sendRegistryCert(masters); err != nil { + return err + } + cmds := k.Command(ControllerRole) + if cmds == nil { + return fmt.Errorf("failed to get join master command") + } + + for _, master := range masters { + logrus.Infof("Start to join %s as master", master) + + masterCmds := k.JoinMasterCommands(cmds) + client, err := k.getHostSSHClient(master) + if err != nil { + return err + } + + if client.(*ssh.SSH).User != common.ROOT { + masterCmds = append(masterCmds, RemoteNonRootCopyKubeConfig) + } + + if err := client.CmdAsync(master, masterCmds...); err != nil { + return fmt.Errorf("failed to exec command(%s) on master(%s): %v", cmds, master, err) + } + + logrus.Infof("Succeeded in joining %s as master", master) + } + return nil +} + +func (k *Runtime) JoinMasterCommands(cmds []string) []string { + cmdAddRegistryHosts := k.addRegistryDomainToHosts() + if k.RegConfig.Domain != SeaHub { + cmdAddSeaHubHosts := fmt.Sprintf(RemoteAddEtcHosts, k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub) + cmdAddRegistryHosts = fmt.Sprintf("%s && %s", cmdAddRegistryHosts, cmdAddSeaHubHosts) + } + joinCommands := []string{cmdAddRegistryHosts} + if k.RegConfig.Username != "" && k.RegConfig.Password != "" { + joinCommands = append(joinCommands, k.GenLoginCommand()) + } + + return append(joinCommands, cmds...) +} diff --git a/pkg/runtime/k0s/join_nodes.go b/pkg/runtime/k0s/join_nodes.go new file mode 100644 index 00000000000..b6c8202bed5 --- /dev/null +++ b/pkg/runtime/k0s/join_nodes.go @@ -0,0 +1,72 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k0s + +import ( + "context" + "fmt" + "net" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +func (k *Runtime) joinNodes(nodes []net.IP) error { + if len(nodes) == 0 { + return nil + } + if err := k.WaitSSHReady(6, nodes...); err != nil { + return errors.Wrap(err, "join nodes wait for ssh ready time out") + } + if err := k.sendRegistryCert(nodes); err != nil { + return err + } + if err := k.CopyJoinToken(WorkerRole, nodes); err != nil { + return err + } + addRegistryHostsAndLogin := k.addRegistryDomainToHosts() + if k.RegConfig.Domain != SeaHub { + addSeaHubHost := fmt.Sprintf(RemoteAddEtcHosts, k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub) + addRegistryHostsAndLogin = fmt.Sprintf("%s && %s", addRegistryHostsAndLogin, addSeaHubHost) + } + if k.RegConfig.Username != "" && k.RegConfig.Password != "" { + addRegistryHostsAndLogin = fmt.Sprintf("%s && %s", addRegistryHostsAndLogin, k.GenLoginCommand()) + } + cmds := k.Command(WorkerRole) + if cmds == nil { + return fmt.Errorf("failed to get join node command") + } + + eg, _ := errgroup.WithContext(context.Background()) + for _, node := range nodes { + node := node + eg.Go(func() error { + logrus.Infof("Start to join %s as worker", node) + + nodeCmds := append([]string{addRegistryHostsAndLogin}, cmds...) + ssh, err := k.getHostSSHClient(node) + if err != nil { + return fmt.Errorf("failed to join node %s: %v", node, err) + } + if err := ssh.CmdAsync(node, nodeCmds...); err != nil { + return fmt.Errorf("failed to join node %s: %v", node, err) + } + logrus.Infof("Succeeded in joining %s as worker", node) + return err + }) + } + return eg.Wait() +} diff --git a/pkg/runtime/k0s/registry.go b/pkg/runtime/k0s/registry.go index 4a1f14b4eac..a4da9e39be7 100644 --- a/pkg/runtime/k0s/registry.go +++ b/pkg/runtime/k0s/registry.go @@ -26,6 +26,8 @@ const ( ContainerdLoginCommand = "nerdctl login -u %s -p %s %s" DefaultRegistryHtPasswdFile = "registry_htpasswd" DockerCertDir = "/etc/docker/certs.d" + DeleteRegistryCommand = "((! nerdctl ps -a 2>/dev/null |grep %[1]s) || (nerdctl stop %[1]s && nerdctl rmi -f %[1]s))" + RegistryName = "sealer-registry" ) // sendRegistryCertAndKey send registry cert to Master0 host. path like: /var/lib/sealer/data/my-k0s-cluster/certs @@ -100,3 +102,12 @@ func (k *Runtime) SendRegistryCert(host []net.IP) error { } return k.sendRegistryCert(host) } + +func (k *Runtime) DeleteRegistry() error { + ssh, err := k.getHostSSHClient(k.RegConfig.IP) + if err != nil { + return fmt.Errorf("failed to delete registry: %v", err) + } + + return ssh.CmdAsync(k.RegConfig.IP, fmt.Sprintf(DeleteRegistryCommand, RegistryName)) +} diff --git a/pkg/runtime/k0s/reset.go b/pkg/runtime/k0s/reset.go new file mode 100644 index 00000000000..3f84f95cffd --- /dev/null +++ b/pkg/runtime/k0s/reset.go @@ -0,0 +1,71 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k0s + +import ( + "context" + "fmt" + "net" + + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +func (k *Runtime) reset() error { + k.resetNodes(k.cluster.GetNodeIPList()) + k.resetMasters(k.cluster.GetMasterIPList()) + + return k.DeleteRegistry() +} + +func (k *Runtime) resetNodes(nodes []net.IP) { + eg, _ := errgroup.WithContext(context.Background()) + for _, node := range nodes { + node := node + eg.Go(func() error { + if err := k.resetNode(node); err != nil { + logrus.Errorf("failed to delete node %s: %v", node, err) + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return + } +} + +func (k *Runtime) resetMasters(nodes []net.IP) { + for _, node := range nodes { + if err := k.resetNode(node); err != nil { + logrus.Errorf("failed to delete master(%s): %v", node, err) + } + } +} + +func (k *Runtime) resetNode(node net.IP) error { + ssh, err := k.getHostSSHClient(node) + if err != nil { + return fmt.Errorf("failed to reset node: %v", err) + } + if err := ssh.CmdAsync(node, fmt.Sprintf(RemoteCleanMasterOrNode, DefaultK0sConfigPath, ExternalCRI), + RemoveKubeConfig, + fmt.Sprintf(RemoteRemoveEtcHost, SeaHub), + fmt.Sprintf(RemoteRemoveEtcHost, k.RegConfig.Domain), + fmt.Sprintf(RemoteRemoveRegistryCerts, k.RegConfig.Domain), + fmt.Sprintf(RemoteRemoveRegistryCerts, SeaHub)); err != nil { + return err + } + return nil +} diff --git a/pkg/runtime/k0s/runtime.go b/pkg/runtime/k0s/runtime.go index f4080302135..1af6d235a4d 100644 --- a/pkg/runtime/k0s/runtime.go +++ b/pkg/runtime/k0s/runtime.go @@ -17,12 +17,16 @@ package k0s import ( "context" "fmt" + "net" + "strings" + "time" "github.com/sealerio/sealer/common" "github.com/sealerio/sealer/pkg/registry" "github.com/sealerio/sealer/pkg/runtime" v2 "github.com/sealerio/sealer/types/api/v2" + "github.com/sealerio/sealer/utils" "github.com/sealerio/sealer/utils/platform" "github.com/sealerio/sealer/utils/ssh" @@ -38,48 +42,60 @@ type Runtime struct { RegConfig *registry.Config } +var ForceDelete bool + func (k *Runtime) Init() error { return k.init() } func (k *Runtime) Upgrade() error { - //TODO implement me - panic("implement me") + return k.upgrade() } func (k *Runtime) Reset() error { - //TODO implement me - panic("implement me") + logrus.Infof("Start to delete cluster: master %s, node %s", k.cluster.GetMasterIPList(), k.cluster.GetNodeIPList()) + if err := k.confirmDeleteNodes(); err != nil { + return err + } + return k.reset() } func (k *Runtime) JoinMasters(newMastersIPList []net.IP) error { - //TODO implement me - panic("implement me") + if len(newMastersIPList) != 0 { + logrus.Infof("%s will be added as master", newMastersIPList) + } + return k.joinMasters(newMastersIPList) } func (k *Runtime) JoinNodes(newNodesIPList []net.IP) error { - //TODO implement me - panic("implement me") + if len(newNodesIPList) != 0 { + logrus.Infof("%s will be added as worker", newNodesIPList) + } + return k.joinNodes(newNodesIPList) } func (k *Runtime) DeleteMasters(mastersIPList []net.IP) error { - //TODO implement me - panic("implement me") + if len(mastersIPList) != 0 { + logrus.Infof("master %s will be deleted", mastersIPList) + if err := k.confirmDeleteNodes(); err != nil { + return err + } + } + return k.deleteMasters(mastersIPList) } func (k *Runtime) DeleteNodes(nodesIPList []net.IP) error { - //TODO implement me - panic("implement me") + if len(nodesIPList) != 0 { + logrus.Infof("worker %s will be deleted", nodesIPList) + if err := k.confirmDeleteNodes(); err != nil { + return err + } + } + return k.deleteNodes(nodesIPList) } func (k *Runtime) GetClusterMetadata() (*runtime.Metadata, error) { - //TODO implement me - panic("implement me") -} - -func (k *Runtime) UpdateCert(certs []string) error { - //TODO implement me - panic("implement me") + return k.getClusterMetadata() } // NewK0sRuntime arg "clusterConfig" is the k0s config file under etc/${ant_name.yaml}, runtime need read k0s config from it @@ -158,3 +174,114 @@ func (k *Runtime) sendFileToHosts(Hosts []net.IP, src, dst string) error { } return eg.Wait() } + +func (k *Runtime) WaitSSHReady(tryTimes int, hosts ...net.IP) error { + eg, _ := errgroup.WithContext(context.Background()) + for _, h := range hosts { + host := h + eg.Go(func() error { + for i := 0; i < tryTimes; i++ { + sshClient, err := k.getHostSSHClient(host) + if err != nil { + return err + } + err = sshClient.Ping(host) + if err == nil { + return nil + } + time.Sleep(time.Duration(i) * time.Second) + } + return fmt.Errorf("wait for [%s] ssh ready timeout, ensure that the IP address or password is correct", host) + }) + } + return eg.Wait() +} + +func (k *Runtime) CopyJoinToken(role string, hosts []net.IP) error { + var joinCertPath string + ssh, err := k.getHostSSHClient(k.cluster.GetMaster0IP()) + if err != nil { + return err + } + switch role { + case ControllerRole: + joinCertPath = DefaultK0sControllerJoin + case WorkerRole: + joinCertPath = DefaultK0sWorkerJoin + default: + joinCertPath = DefaultK0sWorkerJoin + } + + eg, _ := errgroup.WithContext(context.Background()) + for _, host := range hosts { + host := host + eg.Go(func() error { + return ssh.Copy(host, joinCertPath, joinCertPath) + }) + } + return nil +} + +func (k *Runtime) Command(role string) []string { + cmds := map[string][]string{ + ControllerRole: {"mkdir -r /etc/k0s", fmt.Sprintf("k0s config create > %s", DefaultK0sConfigPath), + fmt.Sprintf("sed -i '/ images/ a\\ repository: \"%s:%s\"' %s", k.RegConfig.Domain, k.RegConfig.Port, DefaultK0sConfigPath), + fmt.Sprintf("k0s install controller --token-file %s -c %s", + DefaultK0sControllerJoin, DefaultK0sConfigPath), + "k0s start", + }, + WorkerRole: {fmt.Sprintf("k0s install worker --cri-socket %s --token-file %s", ExternalCRI, DefaultK0sWorkerJoin), + "k0s start"}, + } + + v, ok := cmds[role] + if !ok { + logrus.Errorf("failed to get k0s command: %v", cmds) + return nil + } + return v +} + +func (k *Runtime) confirmDeleteNodes() error { + if !ForceDelete { + if pass, err := utils.ConfirmOperation("Are you sure to delete these nodes? "); err != nil { + return err + } else if !pass { + return fmt.Errorf("exit the operation of delete these nodes") + } + } + return nil +} + +//CmdToString is in host exec cmd and replace to spilt str +func (k *Runtime) CmdToString(host net.IP, cmd, split string) (string, error) { + ssh, err := k.getHostSSHClient(host) + if err != nil { + return "", fmt.Errorf("failed to get ssh clientof host(%s): %v", host, err) + } + return ssh.CmdToString(host, cmd, split) +} + +func (k *Runtime) getClusterMetadata() (*runtime.Metadata, error) { + metadata := &runtime.Metadata{} + + version, err := k.getKubeVersion() + if err != nil { + return metadata, err + } + metadata.Version = version + metadata.ClusterRuntime = RuntimeFlag + return metadata, nil +} + +func (k *Runtime) getKubeVersion() (string, error) { + ssh, err := k.getHostSSHClient(k.cluster.GetMaster0IP()) + if err != nil { + return "", err + } + bytes, err := ssh.Cmd(k.cluster.GetMaster0IP(), VersionCmd) + if err != nil { + return "", err + } + return strings.Split(string(bytes), "+")[0], nil +} diff --git a/pkg/runtime/k0s/upgrade.go b/pkg/runtime/k0s/upgrade.go new file mode 100644 index 00000000000..068db9106bd --- /dev/null +++ b/pkg/runtime/k0s/upgrade.go @@ -0,0 +1,97 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k0s + +import ( + "fmt" + "net" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +const ( + chmodCmd = `chmod +x %s/*` + mvCmd = `mv %s/* /usr/bin` + getNodeNameCmd = `$(uname -n | tr '[A-Z]' '[a-z]')` + drainCmd = `kubectl drain ` + getNodeNameCmd + ` --ignore-daemonsets` + upgradeCmd = `k0s stop && k0s start` + uncordonCmd = `kubectl uncordon ` + getNodeNameCmd +) + +func (k *Runtime) upgrade() error { + var err error + binPath := filepath.Join(k.getRootfs(), `bin`) + + err = k.upgradeMasters([]net.IP{k.cluster.GetMaster0IP()}, binPath) + if err != nil { + return err + } + err = k.upgradeMasters(k.cluster.GetMasterIPList()[1:], binPath) + if err != nil { + return err + } + err = k.upgradeNodes(k.cluster.GetNodeIPList(), binPath) + if err != nil { + return err + } + return nil +} + +func (k *Runtime) upgradeMasters(IPs []net.IP, binPath string) error { + var cmds = []string{ + fmt.Sprintf(chmodCmd, binPath), + fmt.Sprintf(mvCmd, binPath), + fmt.Sprintf("%s %s", drainCmd, "--delete-emptydir-data"), + upgradeCmd, + uncordonCmd, + } + + for _, ip := range IPs { + logrus.Infof("Start to upgrade master %s", ip) + + ssh, err := k.getHostSSHClient(ip) + if err != nil { + return fmt.Errorf("failed to get master ssh client: %v", err) + } + if err := ssh.CmdAsync(ip, cmds...); err != nil { + return err + } + } + + return nil +} + +func (k *Runtime) upgradeNodes(IPs []net.IP, binpath string) error { + var nodeCmds = []string{ + fmt.Sprintf(chmodCmd, binpath), + fmt.Sprintf(mvCmd, binpath), + upgradeCmd, + } + var err error + for _, ip := range IPs { + logrus.Infof("Start to upgrade node %s", ip) + + ssh, err := k.getHostSSHClient(ip) + if err != nil { + return fmt.Errorf("failed to get ssh client of host(%s): %v", ip, err) + } + err = ssh.CmdAsync(ip, nodeCmds...) + if err != nil { + return err + } + } + return err +} From 0691113477035c73219a0d7986c04a456e1e1751 Mon Sep 17 00:00:00 2001 From: starComingup <1225067236@qq.com> Date: Wed, 7 Sep 2022 22:19:53 +0800 Subject: [PATCH 2/5] resolved: log format, cmd in code, simplify code. Signed-off-by: starComingup <1225067236@qq.com> --- pkg/runtime/k0s/delete_masters.go | 82 +++++++++++++++++-------------- pkg/runtime/k0s/delete_nodes.go | 41 ++++++++++------ pkg/runtime/k0s/join_masters.go | 19 ++++--- pkg/runtime/k0s/join_nodes.go | 11 ++++- pkg/runtime/k0s/registry.go | 19 +++---- pkg/runtime/k0s/reset.go | 44 ++++++++++------- pkg/runtime/k0s/upgrade.go | 33 ++++++------- 7 files changed, 146 insertions(+), 103 deletions(-) diff --git a/pkg/runtime/k0s/delete_masters.go b/pkg/runtime/k0s/delete_masters.go index 2d3136ec683..7c3f6a855dd 100644 --- a/pkg/runtime/k0s/delete_masters.go +++ b/pkg/runtime/k0s/delete_masters.go @@ -20,21 +20,12 @@ import ( "net" "strings" + "github.com/sealerio/sealer/pkg/client/k8s" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) -const ( - RemoteCleanMasterOrNode = `if which k0s; then k0s stop && k0s reset --config %s --cri-socket %s;fi && \ -rm -rf /etc/k0s/ -` - RemoveKubeConfig = "rm -rf /usr/bin/kube* && rm -rf ~/.kube/" - RemoveK0sBin = "rm -rf /usr/bin/k0s" - RemoteRemoveEtcHost = "sed -i \"/%s/d\" /etc/hosts" - RemoteRemoveRegistryCerts = "rm -rf " + DockerCertDir + "/%s*" - KubeDeleteNode = "kubectl delete node %s" -) - func (k *Runtime) deleteMasters(masters []net.IP) error { if len(masters) == 0 { return nil @@ -43,13 +34,11 @@ func (k *Runtime) deleteMasters(masters []net.IP) error { for _, master := range masters { master := master eg.Go(func() error { - master := master logrus.Infof("Start to delete master %s", master) if err := k.deleteMaster(master); err != nil { - logrus.Errorf("failed to delete master %s: %v", master, err) - } else { - logrus.Infof("Succeeded in deleting master %s", master) + return fmt.Errorf("failed to delete master %s: %v", master, err) } + logrus.Infof("Succeeded in deleting master %s", master) return nil }) } @@ -61,12 +50,24 @@ func (k *Runtime) deleteMaster(master net.IP) error { if err != nil { return fmt.Errorf("failed to delete master: %v", err) } - remoteCleanCmd := []string{fmt.Sprintf(RemoteCleanMasterOrNode, DefaultK0sConfigPath, ExternalCRI), - fmt.Sprintf(RemoteRemoveEtcHost, SeaHub), - fmt.Sprintf(RemoteRemoveRegistryCerts, k.RegConfig.Domain), - fmt.Sprintf(RemoteRemoveRegistryCerts, SeaHub), - RemoveKubeConfig, - RemoveK0sBin} + /** To delete a node from k0s cluster, following these steps. + STEP1: stop k0s service + STEP2: reset the node with install configuration + STEP3: remove k0s cluster config generate by k0s under /etc/k0s + STEP4: remove private registry config in /etc/host + STEP5: remove bin file such as: kubectl, and remove .kube directory + STEP6: remove k0s bin file + STEP7: delete node though k8s client + */ + remoteCleanCmd := []string{"k0s stop", + fmt.Sprintf("k0s reset --config %s --cri-socket %s", DefaultK0sConfigPath, ExternalCRI), + "rm -rf /etc/k0s/", + fmt.Sprintf("sed -i \"/%s/d\" /etc/hosts", SeaHub), + fmt.Sprintf("sed -i \"/%s/d\" /etc/hosts", k.RegConfig.Domain), + fmt.Sprintf("rm -rf %s /%s*", DockerCertDir, k.RegConfig.Domain), + fmt.Sprintf("rm -rf %s /%s*", DockerCertDir, SeaHub), + "rm -rf /usr/bin/kube* && rm -rf ~/.kube/", + "rm -rf /usr/bin/k0s"} if err := ssh.CmdAsync(master, remoteCleanCmd...); err != nil { return err @@ -81,43 +82,50 @@ func (k *Runtime) deleteMaster(master net.IP) error { } if len(masterIPs) > 0 { - hostname, err := k.isHostName(k.cluster.GetMaster0IP(), master) + hostname, err := k.isHostName(master) if err != nil { return err } - master0SSH, err := k.getHostSSHClient(k.cluster.GetMaster0IP()) + client, err := k8s.Newk8sClient() if err != nil { - return fmt.Errorf("failed to get master0 ssh client: %v", err) + return err } - - if err := master0SSH.CmdAsync(k.cluster.GetMaster0IP(), fmt.Sprintf(KubeDeleteNode, strings.TrimSpace(hostname))); err != nil { - return fmt.Errorf("failed to delete node %s: %v", hostname, err) + if err := client.DeleteNode(hostname); err != nil { + return err } } return nil } -func (k *Runtime) isHostName(master, host net.IP) (string, error) { - hostString, err := k.CmdToString(master, "kubectl get nodes | grep -v NAME | awk '{print $1}'", ",") +func (k *Runtime) isHostName(host net.IP) (string, error) { + client, err := k8s.Newk8sClient() + if err != nil { + return "", err + } + nodeList, err := client.ListNodes() if err != nil { return "", err } + var hosts []string + for _, node := range nodeList.Items { + hosts = append(hosts, node.GetName()) + } + hostName, err := k.CmdToString(host, "hostname", "") if err != nil { return "", err } - hosts := strings.Split(hostString, ",") + var name string for _, h := range hosts { if strings.TrimSpace(h) == "" { continue - } else { - hh := strings.ToLower(h) - fromH := strings.ToLower(hostName) - if hh == fromH { - name = h - break - } + } + hh := strings.ToLower(h) + fromH := strings.ToLower(hostName) + if hh == fromH { + name = h + break } } return name, nil diff --git a/pkg/runtime/k0s/delete_nodes.go b/pkg/runtime/k0s/delete_nodes.go index 3d5436710d5..58df9825ec4 100644 --- a/pkg/runtime/k0s/delete_nodes.go +++ b/pkg/runtime/k0s/delete_nodes.go @@ -20,6 +20,8 @@ import ( "net" "strings" + "github.com/sealerio/sealer/pkg/client/k8s" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) @@ -32,11 +34,11 @@ func (k *Runtime) deleteNodes(nodes []net.IP) error { for _, node := range nodes { node := node eg.Go(func() error { - logrus.Infof("Start to delete worker %s", node) + logrus.Infof("Start to delete node %s", node) if err := k.deleteNode(node); err != nil { return fmt.Errorf("failed to delete node %s: %v", node, err) } - logrus.Infof("Succeeded in deleting worker %s", node) + logrus.Infof("Succeeded in deleting node %s", node) return nil }) } @@ -48,29 +50,40 @@ func (k *Runtime) deleteNode(node net.IP) error { if err != nil { return fmt.Errorf("failed to delete node: %v", err) } - remoteCleanCmds := []string{fmt.Sprintf(RemoteCleanMasterOrNode, DefaultK0sConfigPath, ExternalCRI), - fmt.Sprintf(RemoteRemoveEtcHost, k.RegConfig.Domain), - fmt.Sprintf(RemoteRemoveEtcHost, SeaHub), - fmt.Sprintf(RemoteRemoveRegistryCerts, k.RegConfig.Domain), - fmt.Sprintf(RemoteRemoveRegistryCerts, SeaHub), - RemoveKubeConfig, - RemoveK0sBin} + /** To delete a node from k0s cluster, following these steps. + STEP1: stop k0s service + STEP2: reset the node with install configuration + STEP3: remove k0s cluster config generate by k0s under /etc/k0s + STEP4: remove private registry config in /etc/host + STEP5: remove bin file such as: kubectl, and remove .kube directory + STEP6: remove k0s bin file. + STEP7: delete node though k8s client + */ + remoteCleanCmds := []string{"k0s stop", + fmt.Sprintf("k0s reset --config %s --cri-socket %s", DefaultK0sConfigPath, ExternalCRI), + "rm -rf /etc/k0s/", + fmt.Sprintf("sed -i \"/%s/d\" /etc/hosts", SeaHub), + fmt.Sprintf("sed -i \"/%s/d\" /etc/hosts", k.RegConfig.Domain), + fmt.Sprintf("rm -rf %s /%s*", DockerCertDir, k.RegConfig.Domain), + fmt.Sprintf("rm -rf %s /%s*", DockerCertDir, SeaHub), + "rm -rf /usr/bin/kube* && rm -rf ~/.kube/", + "rm -rf /usr/bin/k0s"} if err := ssh.CmdAsync(node, remoteCleanCmds...); err != nil { return err } //remove node if len(k.cluster.GetMasterIPList()) > 0 { - hostname, err := k.isHostName(k.cluster.GetMaster0IP(), node) + hostname, err := k.isHostName(node) if err != nil { return err } - ssh, err := k.getHostSSHClient(k.cluster.GetMaster0IP()) + client, err := k8s.Newk8sClient() if err != nil { - return fmt.Errorf("failed to get master0 ssh client(%s): %v", k.cluster.GetMaster0IP(), err) + return err } - if err := ssh.CmdAsync(k.cluster.GetMaster0IP(), fmt.Sprintf(KubeDeleteNode, strings.TrimSpace(hostname))); err != nil { - return fmt.Errorf("failed to delete node %s: %v", hostname, err) + if err := client.DeleteNode(strings.TrimSpace(hostname)); err != nil { + return err } } diff --git a/pkg/runtime/k0s/join_masters.go b/pkg/runtime/k0s/join_masters.go index 9cae23d7e0e..266d8375d39 100644 --- a/pkg/runtime/k0s/join_masters.go +++ b/pkg/runtime/k0s/join_masters.go @@ -25,8 +25,6 @@ import ( "github.com/sirupsen/logrus" ) -const RemoteNonRootCopyKubeConfig = `rm -rf ${HOME}/.kube/config && mkdir -p ${HOME}/.kube && cp /var/lib/k0s/pki/admin.conf ${HOME}/.kube/config && chown $(id -u):$(id -g) ${HOME}/.kube/config` - func (k *Runtime) joinMasters(masters []net.IP) error { if len(masters) == 0 { return nil @@ -34,10 +32,19 @@ func (k *Runtime) joinMasters(masters []net.IP) error { if err := k.WaitSSHReady(6, masters...); err != nil { return errors.Wrap(err, "join masters wait for ssh ready time out") } - if err := k.CopyJoinToken(ControllerRole, masters); err != nil { + /**To join a node, following these steps. + STEP1: send private registry cert and add registry info into node + STEP2: copy k0s join token + STEP3: use k0s command to join node with master role. + STEP4: k0s create default config + STEP5: modify the private image repository field and so on in k0s config + STEP6: join node with token + STEP7: start the k0scontroller.service + */ + if err := k.sendRegistryCert(masters); err != nil { return err } - if err := k.sendRegistryCert(masters); err != nil { + if err := k.CopyJoinToken(ControllerRole, masters); err != nil { return err } cmds := k.Command(ControllerRole) @@ -55,7 +62,7 @@ func (k *Runtime) joinMasters(masters []net.IP) error { } if client.(*ssh.SSH).User != common.ROOT { - masterCmds = append(masterCmds, RemoteNonRootCopyKubeConfig) + masterCmds = append(masterCmds, "rm -rf ${HOME}/.kube/config && mkdir -p ${HOME}/.kube && cp /var/lib/k0s/pki/admin.conf ${HOME}/.kube/config && chown $(id -u):$(id -g) ${HOME}/.kube/config") } if err := client.CmdAsync(master, masterCmds...); err != nil { @@ -70,7 +77,7 @@ func (k *Runtime) joinMasters(masters []net.IP) error { func (k *Runtime) JoinMasterCommands(cmds []string) []string { cmdAddRegistryHosts := k.addRegistryDomainToHosts() if k.RegConfig.Domain != SeaHub { - cmdAddSeaHubHosts := fmt.Sprintf(RemoteAddEtcHosts, k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub) + cmdAddSeaHubHosts := fmt.Sprintf("cat /etc/hosts | grep '%s' || echo '%s' >> /etc/hosts", k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub) cmdAddRegistryHosts = fmt.Sprintf("%s && %s", cmdAddRegistryHosts, cmdAddSeaHubHosts) } joinCommands := []string{cmdAddRegistryHosts} diff --git a/pkg/runtime/k0s/join_nodes.go b/pkg/runtime/k0s/join_nodes.go index b6c8202bed5..9ca77c1b177 100644 --- a/pkg/runtime/k0s/join_nodes.go +++ b/pkg/runtime/k0s/join_nodes.go @@ -31,6 +31,13 @@ func (k *Runtime) joinNodes(nodes []net.IP) error { if err := k.WaitSSHReady(6, nodes...); err != nil { return errors.Wrap(err, "join nodes wait for ssh ready time out") } + /**To join a node, following these steps. + STEP1: send private registry cert and add registry info into node + STEP2: copy k0s join token + STEP3: use k0s command to join node with worker role. + STEP4: join node with token + STEP5: start the k0sworker.service + */ if err := k.sendRegistryCert(nodes); err != nil { return err } @@ -39,7 +46,7 @@ func (k *Runtime) joinNodes(nodes []net.IP) error { } addRegistryHostsAndLogin := k.addRegistryDomainToHosts() if k.RegConfig.Domain != SeaHub { - addSeaHubHost := fmt.Sprintf(RemoteAddEtcHosts, k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub) + addSeaHubHost := fmt.Sprintf("cat /etc/hosts | grep '%s' || echo '%s' >> /etc/hosts", k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub) addRegistryHostsAndLogin = fmt.Sprintf("%s && %s", addRegistryHostsAndLogin, addSeaHubHost) } if k.RegConfig.Username != "" && k.RegConfig.Password != "" { @@ -59,7 +66,7 @@ func (k *Runtime) joinNodes(nodes []net.IP) error { nodeCmds := append([]string{addRegistryHostsAndLogin}, cmds...) ssh, err := k.getHostSSHClient(node) if err != nil { - return fmt.Errorf("failed to join node %s: %v", node, err) + return fmt.Errorf("failed to get node ssh client %s: %v", node, err) } if err := ssh.CmdAsync(node, nodeCmds...); err != nil { return fmt.Errorf("failed to join node %s: %v", node, err) diff --git a/pkg/runtime/k0s/registry.go b/pkg/runtime/k0s/registry.go index a4da9e39be7..c34a7eef5ed 100644 --- a/pkg/runtime/k0s/registry.go +++ b/pkg/runtime/k0s/registry.go @@ -22,11 +22,8 @@ import ( const ( SeaHub = "sea.hub" - RemoteAddEtcHosts = "cat /etc/hosts |grep '%s' || echo '%s' >> /etc/hosts" - ContainerdLoginCommand = "nerdctl login -u %s -p %s %s" DefaultRegistryHtPasswdFile = "registry_htpasswd" DockerCertDir = "/etc/docker/certs.d" - DeleteRegistryCommand = "((! nerdctl ps -a 2>/dev/null |grep %[1]s) || (nerdctl stop %[1]s && nerdctl rmi -f %[1]s))" RegistryName = "sealer-registry" ) @@ -47,7 +44,7 @@ func (k *Runtime) sendRegistryCert(host []net.IP) error { func (k *Runtime) addRegistryDomainToHosts() (host string) { content := fmt.Sprintf("%s %s", k.RegConfig.IP.String(), k.RegConfig.Domain) - return fmt.Sprintf(RemoteAddEtcHosts, content, content) + return fmt.Sprintf("cat /etc/hosts | grep '%s' || echo '%s' >> /etc/hosts", content, content) } // ApplyRegistryOnMaster0 Only use this for init, due to the initiation operations. @@ -67,10 +64,10 @@ func (k *Runtime) ApplyRegistryOnMaster0() error { return err } } - initRegistry := fmt.Sprintf("cd %s/scripts && ./init-registry.sh %s %s %s", k.getRootfs(), k.RegConfig.Port, fmt.Sprintf("%s/registry", k.getRootfs()), k.RegConfig.Domain) + initRegistry := k.initRegistryCmd() addRegistryHosts := k.addRegistryDomainToHosts() if k.RegConfig.Domain != SeaHub { - addSeaHubHosts := fmt.Sprintf(RemoteAddEtcHosts, k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub) + addSeaHubHosts := fmt.Sprintf("cat /etc/hosts | grep '%s' || echo '%s' >> /etc/hosts", k.RegConfig.IP.String()+" "+SeaHub, k.RegConfig.IP.String()+" "+SeaHub) addRegistryHosts = fmt.Sprintf("%s && %s", addRegistryHosts, addSeaHubHosts) } if err = ssh.CmdAsync(k.RegConfig.IP, initRegistry); err != nil { @@ -87,8 +84,8 @@ func (k *Runtime) ApplyRegistryOnMaster0() error { func (k *Runtime) GenLoginCommand() string { return fmt.Sprintf("%s && %s", - fmt.Sprintf(ContainerdLoginCommand, k.RegConfig.Username, k.RegConfig.Password, k.RegConfig.Domain+":"+k.RegConfig.Port), - fmt.Sprintf(ContainerdLoginCommand, k.RegConfig.Username, k.RegConfig.Password, SeaHub+":"+k.RegConfig.Port)) + fmt.Sprintf("nerdctl login -u %s -p %s %s", k.RegConfig.Username, k.RegConfig.Password, k.RegConfig.Domain+":"+k.RegConfig.Port), + fmt.Sprintf("nerdctl login -u %s -p %s %s", k.RegConfig.Username, k.RegConfig.Password, SeaHub+":"+k.RegConfig.Port)) } func (k *Runtime) GenerateRegistryCert() error { @@ -109,5 +106,9 @@ func (k *Runtime) DeleteRegistry() error { return fmt.Errorf("failed to delete registry: %v", err) } - return ssh.CmdAsync(k.RegConfig.IP, fmt.Sprintf(DeleteRegistryCommand, RegistryName)) + return ssh.CmdAsync(k.RegConfig.IP, fmt.Sprintf("((! nerdctl ps -a 2>/dev/null |grep %[1]s) || (nerdctl stop %[1]s && nerdctl rmi -f %[1]s))", RegistryName)) +} + +func (k *Runtime) initRegistryCmd() string { + return fmt.Sprintf("cd %s/scripts && ./init-registry.sh %s %s %s", k.getRootfs(), k.RegConfig.Port, fmt.Sprintf("%s/registry", k.getRootfs()), k.RegConfig.Domain) } diff --git a/pkg/runtime/k0s/reset.go b/pkg/runtime/k0s/reset.go index 3f84f95cffd..ea4895f1fff 100644 --- a/pkg/runtime/k0s/reset.go +++ b/pkg/runtime/k0s/reset.go @@ -19,52 +19,62 @@ import ( "fmt" "net" - "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) func (k *Runtime) reset() error { - k.resetNodes(k.cluster.GetNodeIPList()) - k.resetMasters(k.cluster.GetMasterIPList()) + if err := k.resetNodes(k.cluster.GetNodeIPList()); err != nil { + return err + } + if err := k.resetMasters(k.cluster.GetMasterIPList()); err != nil { + return err + } return k.DeleteRegistry() } -func (k *Runtime) resetNodes(nodes []net.IP) { +func (k *Runtime) resetNodes(nodes []net.IP) error { eg, _ := errgroup.WithContext(context.Background()) for _, node := range nodes { node := node eg.Go(func() error { if err := k.resetNode(node); err != nil { - logrus.Errorf("failed to delete node %s: %v", node, err) + return fmt.Errorf("failed to reset node %s: %v", node, err) } return nil }) } - if err := eg.Wait(); err != nil { - return - } + return eg.Wait() } -func (k *Runtime) resetMasters(nodes []net.IP) { +func (k *Runtime) resetMasters(nodes []net.IP) error { for _, node := range nodes { if err := k.resetNode(node); err != nil { - logrus.Errorf("failed to delete master(%s): %v", node, err) + return fmt.Errorf("failed to reset master %s: %v", node, err) } } + return nil } func (k *Runtime) resetNode(node net.IP) error { ssh, err := k.getHostSSHClient(node) if err != nil { - return fmt.Errorf("failed to reset node: %v", err) + return err } - if err := ssh.CmdAsync(node, fmt.Sprintf(RemoteCleanMasterOrNode, DefaultK0sConfigPath, ExternalCRI), - RemoveKubeConfig, - fmt.Sprintf(RemoteRemoveEtcHost, SeaHub), - fmt.Sprintf(RemoteRemoveEtcHost, k.RegConfig.Domain), - fmt.Sprintf(RemoteRemoveRegistryCerts, k.RegConfig.Domain), - fmt.Sprintf(RemoteRemoveRegistryCerts, SeaHub)); err != nil { + /** To reset a node, do following command one by one: + STEP1: stop k0s service + STEP2: reset the node with install configuration + STEP3: remove k0s cluster config generate by k0s under /etc/k0s + STEP4: remove private registry config in /etc/host + */ + if err := ssh.CmdAsync(node, "k0s stop", + fmt.Sprintf("k0s reset --config %s --cri-socket %s", DefaultK0sConfigPath, ExternalCRI), + "rm -rf /etc/k0s/", + "rm -rf /usr/bin/kube* && rm -rf ~/.kube/", + fmt.Sprintf("sed -i \"/%s/d\" /etc/hosts", SeaHub), + fmt.Sprintf("sed -i \"/%s/d\" /etc/hosts", k.RegConfig.Domain), + fmt.Sprintf("rm -rf %s /%s*", DockerCertDir, k.RegConfig.Domain), + fmt.Sprintf("rm -rf %s /%s*", DockerCertDir, SeaHub)); err != nil { return err } return nil diff --git a/pkg/runtime/k0s/upgrade.go b/pkg/runtime/k0s/upgrade.go index 068db9106bd..69e7ecd1051 100644 --- a/pkg/runtime/k0s/upgrade.go +++ b/pkg/runtime/k0s/upgrade.go @@ -22,19 +22,16 @@ import ( "github.com/sirupsen/logrus" ) -const ( - chmodCmd = `chmod +x %s/*` - mvCmd = `mv %s/* /usr/bin` - getNodeNameCmd = `$(uname -n | tr '[A-Z]' '[a-z]')` - drainCmd = `kubectl drain ` + getNodeNameCmd + ` --ignore-daemonsets` - upgradeCmd = `k0s stop && k0s start` - uncordonCmd = `kubectl uncordon ` + getNodeNameCmd -) - func (k *Runtime) upgrade() error { var err error binPath := filepath.Join(k.getRootfs(), `bin`) + /** To upgrade a node + STEP1: prepare a k0s bin file with expected version + STEP2: move k0s bin to /usr/bin + STEP3: stop k0s service + STEP4: start k0s service + */ err = k.upgradeMasters([]net.IP{k.cluster.GetMaster0IP()}, binPath) if err != nil { return err @@ -52,11 +49,10 @@ func (k *Runtime) upgrade() error { func (k *Runtime) upgradeMasters(IPs []net.IP, binPath string) error { var cmds = []string{ - fmt.Sprintf(chmodCmd, binPath), - fmt.Sprintf(mvCmd, binPath), - fmt.Sprintf("%s %s", drainCmd, "--delete-emptydir-data"), - upgradeCmd, - uncordonCmd, + fmt.Sprintf("chmod +x %s/*", binPath), + fmt.Sprintf("mv %s/* /usr/bin", binPath), + "k0s stop", + "k0s start", } for _, ip := range IPs { @@ -74,11 +70,12 @@ func (k *Runtime) upgradeMasters(IPs []net.IP, binPath string) error { return nil } -func (k *Runtime) upgradeNodes(IPs []net.IP, binpath string) error { +func (k *Runtime) upgradeNodes(IPs []net.IP, binPath string) error { var nodeCmds = []string{ - fmt.Sprintf(chmodCmd, binpath), - fmt.Sprintf(mvCmd, binpath), - upgradeCmd, + fmt.Sprintf("chmod +x %s/*", binPath), + fmt.Sprintf("mv %s/* /usr/bin", binPath), + "k0s stop", + "k0s start", } var err error for _, ip := range IPs { From 65a56fc6fe1a1b7bbae69e0cba2bbe294ee49176 Mon Sep 17 00:00:00 2001 From: starComingup <1225067236@qq.com> Date: Thu, 8 Sep 2022 20:02:58 +0800 Subject: [PATCH 3/5] resolved function name confuse, simplify code. Signed-off-by: starComingup <1225067236@qq.com> --- pkg/runtime/k0s/delete_masters.go | 33 ++++++++++++++----------------- pkg/runtime/k0s/delete_nodes.go | 2 +- pkg/runtime/k0s/runtime.go | 1 - 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/pkg/runtime/k0s/delete_masters.go b/pkg/runtime/k0s/delete_masters.go index 7c3f6a855dd..3c2dc17d26e 100644 --- a/pkg/runtime/k0s/delete_masters.go +++ b/pkg/runtime/k0s/delete_masters.go @@ -22,6 +22,7 @@ import ( "github.com/sealerio/sealer/pkg/client/k8s" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) @@ -74,30 +75,26 @@ func (k *Runtime) deleteMaster(master net.IP) error { } // remove master - masterIPs := []net.IP{} - for _, ip := range k.cluster.GetMasterIPList() { - if !ip.Equal(master) { - masterIPs = append(masterIPs, ip) - } + masterExist := len(k.cluster.GetMasterIPList()) > 1 + if !masterExist { + return errors.New("can not delete the last master") } - if len(masterIPs) > 0 { - hostname, err := k.isHostName(master) - if err != nil { - return err - } - client, err := k8s.Newk8sClient() - if err != nil { - return err - } - if err := client.DeleteNode(hostname); err != nil { - return err - } + hostname, err := k.getHostName(master) + if err != nil { + return err + } + client, err := k8s.Newk8sClient() + if err != nil { + return err + } + if err := client.DeleteNode(hostname); err != nil { + return err } return nil } -func (k *Runtime) isHostName(host net.IP) (string, error) { +func (k *Runtime) getHostName(host net.IP) (string, error) { client, err := k8s.Newk8sClient() if err != nil { return "", err diff --git a/pkg/runtime/k0s/delete_nodes.go b/pkg/runtime/k0s/delete_nodes.go index 58df9825ec4..8cb37d1796a 100644 --- a/pkg/runtime/k0s/delete_nodes.go +++ b/pkg/runtime/k0s/delete_nodes.go @@ -74,7 +74,7 @@ func (k *Runtime) deleteNode(node net.IP) error { //remove node if len(k.cluster.GetMasterIPList()) > 0 { - hostname, err := k.isHostName(node) + hostname, err := k.getHostName(node) if err != nil { return err } diff --git a/pkg/runtime/k0s/runtime.go b/pkg/runtime/k0s/runtime.go index 1af6d235a4d..82c14c4020a 100644 --- a/pkg/runtime/k0s/runtime.go +++ b/pkg/runtime/k0s/runtime.go @@ -17,7 +17,6 @@ package k0s import ( "context" "fmt" - "net" "strings" "time" From a4872762e16df1ea5d92a6e42c108f3fa52bc59f Mon Sep 17 00:00:00 2001 From: starComingup <1225067236@qq.com> Date: Mon, 12 Sep 2022 20:15:42 +0800 Subject: [PATCH 4/5] =?UTF-8?q?add=20runtime=20entrance,=20diff=20k0s?= =?UTF-8?q?=E3=80=81k8s=E3=80=81k3s=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: starComingup <1225067236@qq.com> --- apply/driver/local.go | 11 ++++------- apply/processor/create.go | 4 ++-- apply/processor/delete.go | 12 +++++------- apply/processor/gen.go | 17 ++++++----------- apply/processor/scale.go | 9 +++------ apply/processor/utils.go | 39 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 59 insertions(+), 33 deletions(-) create mode 100644 apply/processor/utils.go diff --git a/apply/driver/local.go b/apply/driver/local.go index f0cecade0c8..d768af353b9 100644 --- a/apply/driver/local.go +++ b/apply/driver/local.go @@ -18,18 +18,15 @@ import ( "fmt" "net" - imagecommon "github.com/sealerio/sealer/pkg/define/options" - - "github.com/sealerio/sealer/pkg/auth" - "github.com/sealerio/sealer/pkg/imageengine" - "github.com/sealerio/sealer/apply/processor" "github.com/sealerio/sealer/common" + "github.com/sealerio/sealer/pkg/auth" "github.com/sealerio/sealer/pkg/client/k8s" "github.com/sealerio/sealer/pkg/clusterfile" + imagecommon "github.com/sealerio/sealer/pkg/define/options" "github.com/sealerio/sealer/pkg/filesystem/clusterimage" + "github.com/sealerio/sealer/pkg/imageengine" "github.com/sealerio/sealer/pkg/runtime" - "github.com/sealerio/sealer/pkg/runtime/kubernetes" v2 "github.com/sealerio/sealer/types/api/v2" "github.com/sealerio/sealer/utils" osi "github.com/sealerio/sealer/utils/os" @@ -237,7 +234,7 @@ func (applier *Applier) Upgrade(upgradeImgName string) error { } func (applier *Applier) upgrade() error { - runtimeInterface, err := kubernetes.NewDefaultRuntime(applier.ClusterDesired, applier.ClusterFile.GetKubeadmConfig()) + runtimeInterface, err := processor.RuntimeChoose(platform.DefaultMountClusterImageDir(applier.ClusterDesired.Name), applier.ClusterDesired, applier.ClusterFile.GetKubeadmConfig()) if err != nil { return fmt.Errorf("failed to init runtime: %v", err) } diff --git a/apply/processor/create.go b/apply/processor/create.go index fa6ecae1562..958b556b998 100644 --- a/apply/processor/create.go +++ b/apply/processor/create.go @@ -30,7 +30,6 @@ import ( "github.com/sealerio/sealer/pkg/guest" "github.com/sealerio/sealer/pkg/plugin" "github.com/sealerio/sealer/pkg/runtime" - "github.com/sealerio/sealer/pkg/runtime/kubernetes" v2 "github.com/sealerio/sealer/types/api/v2" "github.com/sealerio/sealer/utils/net" "github.com/sealerio/sealer/utils/platform" @@ -111,7 +110,8 @@ func (c *CreateProcessor) MountImage(cluster *v2.Cluster) error { if err = c.cloudImageMounter.MountImage(cluster); err != nil { return err } - runTime, err := kubernetes.NewDefaultRuntime(cluster, c.ClusterFile.GetKubeadmConfig()) + //TODO split kubeadm config from cluster file. + runTime, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, c.ClusterFile.GetKubeadmConfig()) if err != nil { return fmt.Errorf("failed to init runtime: %v", err) } diff --git a/apply/processor/delete.go b/apply/processor/delete.go index cf4ee03c720..fd718a9a8cb 100644 --- a/apply/processor/delete.go +++ b/apply/processor/delete.go @@ -17,20 +17,18 @@ package processor import ( "fmt" - common2 "github.com/sealerio/sealer/pkg/define/options" - - "github.com/sealerio/sealer/pkg/imageengine" - "github.com/sealerio/sealer/pkg/registry" - "github.com/sealerio/sealer/common" "github.com/sealerio/sealer/pkg/clusterfile" + common2 "github.com/sealerio/sealer/pkg/define/options" "github.com/sealerio/sealer/pkg/filesystem" "github.com/sealerio/sealer/pkg/filesystem/cloudfilesystem" "github.com/sealerio/sealer/pkg/filesystem/clusterimage" + "github.com/sealerio/sealer/pkg/imageengine" "github.com/sealerio/sealer/pkg/plugin" - "github.com/sealerio/sealer/pkg/runtime/kubernetes" + "github.com/sealerio/sealer/pkg/registry" v2 "github.com/sealerio/sealer/types/api/v2" utilsnet "github.com/sealerio/sealer/utils/net" + "github.com/sealerio/sealer/utils/platform" ) type DeleteProcessor struct { @@ -40,7 +38,7 @@ type DeleteProcessor struct { } func (d *DeleteProcessor) Reset(cluster *v2.Cluster) error { - runTime, err := kubernetes.NewDefaultRuntime(cluster, d.ClusterFile.GetKubeadmConfig()) + runTime, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, d.ClusterFile.GetKubeadmConfig()) if err != nil { return fmt.Errorf("failed to init runtime: %v", err) } diff --git a/apply/processor/gen.go b/apply/processor/gen.go index 39fc2c169cc..31dae2d1ddd 100644 --- a/apply/processor/gen.go +++ b/apply/processor/gen.go @@ -19,22 +19,17 @@ package processor import ( "fmt" "net" - - "github.com/sealerio/sealer/pkg/auth" - - "github.com/sealerio/sealer/pkg/define/options" - "strconv" - "github.com/sealerio/sealer/pkg/imageengine" - - "github.com/sealerio/sealer/pkg/registry" - "github.com/sealerio/sealer/common" + "github.com/sealerio/sealer/pkg/auth" "github.com/sealerio/sealer/pkg/client/k8s" "github.com/sealerio/sealer/pkg/clusterfile" + "github.com/sealerio/sealer/pkg/define/options" "github.com/sealerio/sealer/pkg/filesystem" "github.com/sealerio/sealer/pkg/filesystem/clusterimage" + "github.com/sealerio/sealer/pkg/imageengine" + "github.com/sealerio/sealer/pkg/registry" "github.com/sealerio/sealer/pkg/runtime/kubernetes" v2 "github.com/sealerio/sealer/types/api/v2" utilsnet "github.com/sealerio/sealer/utils/net" @@ -194,7 +189,7 @@ func (g *GenerateProcessor) MountImage(cluster *v2.Cluster) error { if err = g.ImageMounter.MountImage(cluster); err != nil { return err } - runt, err := kubernetes.NewDefaultRuntime(cluster, nil) + runt, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil) if err != nil { return err } @@ -207,7 +202,7 @@ func (g *GenerateProcessor) UnmountImage(cluster *v2.Cluster) error { } func (g *GenerateProcessor) ApplyRegistry(cluster *v2.Cluster) error { - runt, err := kubernetes.NewDefaultRuntime(cluster, nil) + runt, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil) if err != nil { return err } diff --git a/apply/processor/scale.go b/apply/processor/scale.go index 5f15078eb52..59597f0db83 100644 --- a/apply/processor/scale.go +++ b/apply/processor/scale.go @@ -18,10 +18,6 @@ import ( "fmt" "net" - "github.com/sealerio/sealer/utils/platform" - - "github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm" - "github.com/sealerio/sealer/common" "github.com/sealerio/sealer/pkg/clusterfile" "github.com/sealerio/sealer/pkg/config" @@ -29,8 +25,9 @@ import ( "github.com/sealerio/sealer/pkg/filesystem/cloudfilesystem" "github.com/sealerio/sealer/pkg/plugin" "github.com/sealerio/sealer/pkg/runtime" - "github.com/sealerio/sealer/pkg/runtime/kubernetes" + "github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm" v2 "github.com/sealerio/sealer/types/api/v2" + platform "github.com/sealerio/sealer/utils/platform" ) type ScaleProcessor struct { @@ -74,7 +71,7 @@ func (s *ScaleProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error } func (s *ScaleProcessor) PreProcess(cluster *v2.Cluster) error { - runTime, err := kubernetes.NewDefaultRuntime(cluster, s.KubeadmConfig) + runTime, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, s.ClusterFile.GetKubeadmConfig()) if err != nil { return fmt.Errorf("failed to init default runtime: %v", err) } diff --git a/apply/processor/utils.go b/apply/processor/utils.go new file mode 100644 index 00000000000..1f96cbf9add --- /dev/null +++ b/apply/processor/utils.go @@ -0,0 +1,39 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package processor + +import ( + "github.com/sealerio/sealer/pkg/runtime" + "github.com/sealerio/sealer/pkg/runtime/k0s" + "github.com/sealerio/sealer/pkg/runtime/kubernetes" + "github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm" + v2 "github.com/sealerio/sealer/types/api/v2" +) + +func RuntimeChoose(rootfs string, cluster *v2.Cluster, config *kubeadm.KubeadmConfig) (runtime.Interface, error) { + metadata, err := runtime.LoadMetadata(rootfs) + if err != nil { + return nil, err + } + switch metadata.ClusterRuntime { + case runtime.K8s: + return kubernetes.NewDefaultRuntime(cluster, config) + case runtime.K0s: + return k0s.NewK0sRuntime(cluster) + // Todo case runtime.K3s: + default: + return kubernetes.NewDefaultRuntime(cluster, config) + } +} From 28bd67e8dbf46c1c3d7359becad5bb7b2ac52d3d Mon Sep 17 00:00:00 2001 From: starComingup <1225067236@qq.com> Date: Wed, 14 Sep 2022 21:21:54 +0800 Subject: [PATCH 5/5] resolve conversation and enhanced runtime logic Signed-off-by: starComingup <1225067236@qq.com> --- apply/driver/local.go | 2 +- apply/processor/create.go | 2 +- apply/processor/delete.go | 2 +- apply/processor/gen.go | 4 ++-- apply/processor/scale.go | 16 +++++++++------- apply/processor/utils.go | 16 +++++++++++++++- cmd/sealer/cmd/cluster/apply.go | 8 ++++---- cmd/sealer/cmd/cluster/delete.go | 5 ++--- pkg/runtime/interface.go | 2 ++ pkg/runtime/k0s/reset.go | 13 +++++++++---- pkg/runtime/k0s/runtime.go | 4 +--- pkg/runtime/kubernetes/runtime.go | 4 +--- 12 files changed, 48 insertions(+), 30 deletions(-) diff --git a/apply/driver/local.go b/apply/driver/local.go index d768af353b9..83b179fb1d7 100644 --- a/apply/driver/local.go +++ b/apply/driver/local.go @@ -234,7 +234,7 @@ func (applier *Applier) Upgrade(upgradeImgName string) error { } func (applier *Applier) upgrade() error { - runtimeInterface, err := processor.RuntimeChoose(platform.DefaultMountClusterImageDir(applier.ClusterDesired.Name), applier.ClusterDesired, applier.ClusterFile.GetKubeadmConfig()) + runtimeInterface, err := processor.ChooseRuntime(platform.DefaultMountClusterImageDir(applier.ClusterDesired.Name), applier.ClusterDesired, applier.ClusterFile.GetKubeadmConfig()) if err != nil { return fmt.Errorf("failed to init runtime: %v", err) } diff --git a/apply/processor/create.go b/apply/processor/create.go index 958b556b998..8f28c63fee2 100644 --- a/apply/processor/create.go +++ b/apply/processor/create.go @@ -111,7 +111,7 @@ func (c *CreateProcessor) MountImage(cluster *v2.Cluster) error { return err } //TODO split kubeadm config from cluster file. - runTime, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, c.ClusterFile.GetKubeadmConfig()) + runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, c.ClusterFile.GetKubeadmConfig()) if err != nil { return fmt.Errorf("failed to init runtime: %v", err) } diff --git a/apply/processor/delete.go b/apply/processor/delete.go index fd718a9a8cb..5d6abfb4588 100644 --- a/apply/processor/delete.go +++ b/apply/processor/delete.go @@ -38,7 +38,7 @@ type DeleteProcessor struct { } func (d *DeleteProcessor) Reset(cluster *v2.Cluster) error { - runTime, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, d.ClusterFile.GetKubeadmConfig()) + runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, d.ClusterFile.GetKubeadmConfig()) if err != nil { return fmt.Errorf("failed to init runtime: %v", err) } diff --git a/apply/processor/gen.go b/apply/processor/gen.go index 31dae2d1ddd..5abadb67a01 100644 --- a/apply/processor/gen.go +++ b/apply/processor/gen.go @@ -189,7 +189,7 @@ func (g *GenerateProcessor) MountImage(cluster *v2.Cluster) error { if err = g.ImageMounter.MountImage(cluster); err != nil { return err } - runt, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil) + runt, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil) if err != nil { return err } @@ -202,7 +202,7 @@ func (g *GenerateProcessor) UnmountImage(cluster *v2.Cluster) error { } func (g *GenerateProcessor) ApplyRegistry(cluster *v2.Cluster) error { - runt, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil) + runt, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), cluster, nil) if err != nil { return err } diff --git a/apply/processor/scale.go b/apply/processor/scale.go index 59597f0db83..dae103b38a1 100644 --- a/apply/processor/scale.go +++ b/apply/processor/scale.go @@ -71,14 +71,9 @@ func (s *ScaleProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error } func (s *ScaleProcessor) PreProcess(cluster *v2.Cluster) error { - runTime, err := RuntimeChoose(platform.DefaultMountClusterImageDir(cluster.Name), cluster, s.ClusterFile.GetKubeadmConfig()) - if err != nil { - return fmt.Errorf("failed to init default runtime: %v", err) - } - s.Runtime = runTime s.Config = config.NewConfiguration(platform.DefaultMountClusterImageDir(cluster.Name)) if s.IsScaleUp { - if err = clusterfile.SaveToDisk(cluster, cluster.Name); err != nil { + if err := clusterfile.SaveToDisk(cluster, cluster.Name); err != nil { return err } } @@ -127,7 +122,8 @@ func (s *ScaleProcessor) Delete(cluster *v2.Cluster) error { } func NewScaleProcessor(kubeadmConfig *kubeadm.KubeadmConfig, clusterFile clusterfile.Interface, masterToJoin, masterToDelete, nodeToJoin, nodeToDelete []net.IP) (Processor, error) { - fs, err := filesystem.NewFilesystem(common.DefaultTheClusterRootfsDir(clusterFile.GetCluster().Name)) + cluster := clusterFile.GetCluster() + fs, err := filesystem.NewFilesystem(common.DefaultTheClusterRootfsDir(cluster.Name)) if err != nil { return nil, err } @@ -138,7 +134,13 @@ func NewScaleProcessor(kubeadmConfig *kubeadm.KubeadmConfig, clusterFile cluster up = true } + runTime, err := ChooseRuntime(platform.DefaultMountClusterImageDir(cluster.Name), &cluster, clusterFile.GetKubeadmConfig()) + if err != nil { + return nil, fmt.Errorf("failed to init default runtime: %v", err) + } + return &ScaleProcessor{ + Runtime: runTime, MastersToDelete: masterToDelete, MastersToJoin: masterToJoin, NodesToDelete: nodeToDelete, diff --git a/apply/processor/utils.go b/apply/processor/utils.go index 1f96cbf9add..1c390a1deea 100644 --- a/apply/processor/utils.go +++ b/apply/processor/utils.go @@ -20,13 +20,17 @@ import ( "github.com/sealerio/sealer/pkg/runtime/kubernetes" "github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm" v2 "github.com/sealerio/sealer/types/api/v2" + "github.com/sirupsen/logrus" ) -func RuntimeChoose(rootfs string, cluster *v2.Cluster, config *kubeadm.KubeadmConfig) (runtime.Interface, error) { +func ChooseRuntime(rootfs string, cluster *v2.Cluster, config *kubeadm.KubeadmConfig) (runtime.Interface, error) { metadata, err := runtime.LoadMetadata(rootfs) if err != nil { return nil, err } + + hintInfo(metadata.ClusterRuntime) + switch metadata.ClusterRuntime { case runtime.K8s: return kubernetes.NewDefaultRuntime(cluster, config) @@ -37,3 +41,13 @@ func RuntimeChoose(rootfs string, cluster *v2.Cluster, config *kubeadm.KubeadmCo return kubernetes.NewDefaultRuntime(cluster, config) } } + +// hintInfo will hint end-user which cluster runtime will load and move to using new version ClusterImage. +func hintInfo(clusterRuntime runtime.ClusterRuntime) { + if clusterRuntime != "" { + logrus.Infof("using %s as a cluster runtime.", clusterRuntime) + return + } + logrus.Infof("using k8s as a default cluster runtime.") + logrus.Infof("the old sealer version is deprecated, using new version of sealer and ClusterImage is encouraged!") +} diff --git a/cmd/sealer/cmd/cluster/apply.go b/cmd/sealer/cmd/cluster/apply.go index 850055f4d24..6ce4cfec7d8 100644 --- a/cmd/sealer/cmd/cluster/apply.go +++ b/cmd/sealer/cmd/cluster/apply.go @@ -15,10 +15,10 @@ package cluster import ( - "github.com/sealerio/sealer/pkg/runtime/kubernetes" - "github.com/spf13/cobra" - "github.com/sealerio/sealer/apply" + "github.com/sealerio/sealer/pkg/runtime" + + "github.com/spf13/cobra" ) var clusterFile string @@ -44,6 +44,6 @@ func NewApplyCmd() *cobra.Command { }, } applyCmd.Flags().StringVarP(&clusterFile, "Clusterfile", "f", "Clusterfile", "Clusterfile path to apply a Kubernetes cluster") - applyCmd.Flags().BoolVar(&kubernetes.ForceDelete, "force", false, "force to delete the specified cluster if set true") + applyCmd.Flags().BoolVar(&runtime.ForceDelete, "force", false, "force to delete the specified cluster if set true") return applyCmd } diff --git a/cmd/sealer/cmd/cluster/delete.go b/cmd/sealer/cmd/cluster/delete.go index b22c1533ca2..505cc10e678 100644 --- a/cmd/sealer/cmd/cluster/delete.go +++ b/cmd/sealer/cmd/cluster/delete.go @@ -19,9 +19,8 @@ import ( "github.com/sealerio/sealer/apply" "github.com/sealerio/sealer/common" - "github.com/sealerio/sealer/pkg/clusterfile" - "github.com/sealerio/sealer/pkg/runtime/kubernetes" + "github.com/sealerio/sealer/pkg/runtime" "github.com/spf13/cobra" ) @@ -100,7 +99,7 @@ func NewDeleteCmd() *cobra.Command { deleteCmd.Flags().StringVarP(&deleteClusterFile, "Clusterfile", "f", "", "delete a kubernetes cluster with Clusterfile Annotations") deleteCmd.Flags().StringVarP(&deleteClusterName, "cluster", "c", "", "delete a kubernetes cluster with cluster name") deleteCmd.Flags().StringSliceVarP(&deleteArgs.CustomEnv, "env", "e", []string{}, "set custom environment variables") - deleteCmd.Flags().BoolVar(&kubernetes.ForceDelete, "force", false, "We also can input an --force flag to delete cluster by force") + deleteCmd.Flags().BoolVar(&runtime.ForceDelete, "force", false, "We also can input an --force flag to delete cluster by force") deleteCmd.Flags().BoolP("all", "a", false, "this flags is for delete nodes, if this is true, empty all node ip") return deleteCmd } diff --git a/pkg/runtime/interface.go b/pkg/runtime/interface.go index 022ba96dae4..957ced70a7d 100644 --- a/pkg/runtime/interface.go +++ b/pkg/runtime/interface.go @@ -36,3 +36,5 @@ type Interface interface { // GetClusterMetadata read the rootfs/Metadata file to get some install info for cluster. GetClusterMetadata() (*Metadata, error) } + +var ForceDelete bool diff --git a/pkg/runtime/k0s/reset.go b/pkg/runtime/k0s/reset.go index ea4895f1fff..3791627deb7 100644 --- a/pkg/runtime/k0s/reset.go +++ b/pkg/runtime/k0s/reset.go @@ -48,12 +48,17 @@ func (k *Runtime) resetNodes(nodes []net.IP) error { } func (k *Runtime) resetMasters(nodes []net.IP) error { + eg, _ := errgroup.WithContext(context.Background()) for _, node := range nodes { - if err := k.resetNode(node); err != nil { - return fmt.Errorf("failed to reset master %s: %v", node, err) - } + node := node + eg.Go(func() error { + if err := k.resetNode(node); err != nil { + return fmt.Errorf("failed to reset master %s: %v", node, err) + } + return nil + }) } - return nil + return eg.Wait() } func (k *Runtime) resetNode(node net.IP) error { diff --git a/pkg/runtime/k0s/runtime.go b/pkg/runtime/k0s/runtime.go index 82c14c4020a..6834b90ca93 100644 --- a/pkg/runtime/k0s/runtime.go +++ b/pkg/runtime/k0s/runtime.go @@ -41,8 +41,6 @@ type Runtime struct { RegConfig *registry.Config } -var ForceDelete bool - func (k *Runtime) Init() error { return k.init() } @@ -242,7 +240,7 @@ func (k *Runtime) Command(role string) []string { } func (k *Runtime) confirmDeleteNodes() error { - if !ForceDelete { + if !runtime.ForceDelete { if pass, err := utils.ConfirmOperation("Are you sure to delete these nodes? "); err != nil { return err } else if !pass { diff --git a/pkg/runtime/kubernetes/runtime.go b/pkg/runtime/kubernetes/runtime.go index cb432174fad..5945c214ff7 100644 --- a/pkg/runtime/kubernetes/runtime.go +++ b/pkg/runtime/kubernetes/runtime.go @@ -41,8 +41,6 @@ import ( "golang.org/x/sync/errgroup" ) -var ForceDelete bool - type Config struct { Vlog int VIP string @@ -142,7 +140,7 @@ func (k *Runtime) DeleteNodes(nodesIPList []net.IP) error { } func (k *Runtime) confirmDeleteNodes() error { - if !ForceDelete { + if !runtime.ForceDelete { if pass, err := utils.ConfirmOperation("Are you sure to delete these nodes? "); err != nil { return err } else if !pass {