scale-up new node will not depend on all existing nodes reachable.

Signed-off-by: wb-lyk925458 <[email protected]>

pkg/cluster-runtime/scale.go

@@ -22,14 +22,31 @@ import (
 	"github.com/sealerio/sealer/common"
 	"github.com/sealerio/sealer/pkg/registry"
 	"github.com/sealerio/sealer/pkg/runtime"
+	"github.com/sirupsen/logrus"
 )
 
 func (i *Installer) ScaleUp(newMasters, newWorkers []net.IP) (registry.Driver, runtime.Driver, error) {
 	masters := i.infraDriver.GetHostIPListByRole(common.MASTER)
 	master0 := masters[0]
+	workers := getWorkerIPList(i.infraDriver)
 	registryDeployHosts := []net.IP{master0}
 	all := append(newMasters, newWorkers...)
 	rootfs := i.infraDriver.GetClusterRootfsPath()
+
+	logrus.Debug("check ssh of new nodes")
+	err := CheckNodeSSH(i.infraDriver, append(newMasters, newWorkers...))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if len(newMasters) != 0 {
+		logrus.Debug("check ssh of workers")
+		err = CheckNodeSSH(i.infraDriver, workers)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
 	// set HostAlias
 	if err := i.infraDriver.SetClusterHostAliases(all); err != nil {
 		return nil, nil, err
@@ -119,9 +136,24 @@ func (i *Installer) ScaleUp(newMasters, newWorkers []net.IP) (registry.Driver, r
 func (i *Installer) ScaleDown(mastersToDelete, workersToDelete []net.IP) (registry.Driver, runtime.Driver, error) {
 	masters := i.infraDriver.GetHostIPListByRole(common.MASTER)
 	master0 := masters[0]
+	workers := getWorkerIPList(i.infraDriver)
 	registryDeployHosts := []net.IP{master0}
 	all := append(mastersToDelete, workersToDelete...)
 
+	logrus.Debug("check ssh of nodesToDelete")
+	err := CheckNodeSSH(i.infraDriver, append(mastersToDelete, workersToDelete...))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if len(mastersToDelete) != 0 {
+		logrus.Debug("check ssh of workers")
+		err = CheckNodeSSH(i.infraDriver, workers)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
 	if err := i.runHostHook(PreCleanHost, all); err != nil {
 		return nil, nil, err
 	}

pkg/cluster-runtime/uninstall.go

@@ -28,6 +28,11 @@ func (i *Installer) UnInstall() error {
 	workers := getWorkerIPList(i.infraDriver)
 	all := append(masters, workers...)
 
+	err := CheckNodeSSH(i.infraDriver, all)
+	if err != nil {
+		return err
+	}
+
 	if err := i.runClusterHook(master0, PreUnInstallCluster); err != nil {
 		return err
 	}

pkg/cluster-runtime/utils.go

@@ -71,3 +71,15 @@ func LoadToRegistry(infraDriver infradriver.InfraDriver, distributor imagedistri
 	logrus.Infof("load image success")
 	return nil
 }
+
+func CheckNodeSSH(infraDriver infradriver.InfraDriver, clientHosts []net.IP) error {
+	for i := range clientHosts {
+		n := clientHosts[i]
+		logrus.Debug("checking ssh client of ", n)
+		err := infraDriver.CmdAsync(n, nil, "ls >> /dev/null")
+		if err != nil {
+			return fmt.Errorf("failed to connect node: %s: %v", n, err)
+		}
+	}
+	return nil
+}

pkg/registry/local.go

@@ -74,7 +74,10 @@ func (c *localConfigurator) UninstallFrom(deletedMasters, deletedNodes []net.IP)
 	if len(c.deployHosts) == 0 {
 		return nil
 	}
-
+	// if deletedMasters is nil, means no need to flush workers, just return
+	if len(deletedMasters) == 0 {
+		return nil
+	}
 	// flush ipvs policy on remain nodes
 	return c.configureLvs(c.deployHosts, netutils.RemoveIPs(c.infraDriver.GetHostIPListByRole(common.NODE), deletedNodes))
 }
@@ -147,6 +150,10 @@ func (c *localConfigurator) configureRegistryNetwork(masters, nodes []net.IP) er
 		return err
 	}
 
+	// if masters is nil, means no need to flush old nodes
+	if len(masters) == 0 {
+		return c.configureLvs(c.deployHosts, nodes)
+	}
 	return c.configureLvs(c.deployHosts, c.infraDriver.GetHostIPListByRole(common.NODE))
 }
 
@@ -202,7 +209,7 @@ func (c *localConfigurator) configureLvs(registryHosts, clientHosts []net.IP) er
 		eg.Go(func() error {
 			err := c.infraDriver.CmdAsync(n, nil, ipvsCmd, lvscareStaticCmd)
 			if err != nil {
-				return fmt.Errorf("failed to config nodes lvs policy %s: %v", ipvsCmd, err)
+				return fmt.Errorf("failed to config nodes lvs policy: %s: %v", ipvsCmd, err)
 			}
 
 			err = c.infraDriver.CmdAsync(n, nil, shellcommand.CommandSetHostAlias(c.Domain, vip))