feat: support ha mode for local registry

cmd/sealer/cmd/alpha/alpha.go

@@ -37,6 +37,5 @@ func NewCmdAlpha() *cobra.Command {
 	cmd.AddCommand(NewGenCmd())
 	cmd.AddCommand(NewCheckCmd())
 	cmd.AddCommand(NewSearchCmd())
-	cmd.AddCommand(NewRecoverCmd())
 	return cmd
 }

cmd/sealer/cmd/cluster/apply.go

@@ -19,11 +19,9 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/sealerio/sealer/cmd/sealer/cmd/utils"
-
-	"github.com/sealerio/sealer/cmd/sealer/cmd/types"
-
 	"github.com/pkg/errors"
+	"github.com/sealerio/sealer/cmd/sealer/cmd/types"
+	"github.com/sealerio/sealer/cmd/sealer/cmd/utils"
 	"github.com/sealerio/sealer/common"
 	"github.com/sealerio/sealer/pkg/clusterfile"
 	v12 "github.com/sealerio/sealer/pkg/define/image/v1"
@@ -104,7 +102,7 @@ func NewApplyCmd() *cobra.Command {
 			if client == nil {
 				// no k8s client means to init a new cluster.
 				logrus.Infof("start to create new cluster with image: %s", imageName)
-				return createNewCluster(imageName, infraDriver, imageEngine, cf, applyMode)
+				return createNewCluster(infraDriver, imageEngine, cf, applyMode)
 			}
 
 			currentCluster, err := utils.GetCurrentCluster(client)

cmd/sealer/cmd/cluster/run-app.go

@@ -86,7 +86,7 @@ func NewRunAPPCmd() *cobra.Command {
 }
 
 func installApplication(appImageName string, launchCmds []string, extension v12.ImageExtension,
-	infraDriver infradriver.InfraDriver, imageEngine imageengine.Interface, createMode string) error {
+	infraDriver infradriver.InfraDriver, imageEngine imageengine.Interface, mode string) error {
 	clusterHosts := infraDriver.GetHostIPList()
 
 	clusterHostsPlatform, err := infraDriver.GetHostsPlatform(clusterHosts)
@@ -116,14 +116,8 @@ func installApplication(appImageName string, launchCmds []string, extension v12.
 		return err
 	}
 
-	if createMode == common.ApplyModeLoadImage {
-		logrus.Infof("start to apply with mode(%s)", createMode)
-		reg := infraDriver.GetClusterRegistryConfig()
-		if err = distributor.DistributeRegistry(reg.LocalRegistry.DeployHosts, infraDriver.GetClusterRootfsPath()); err != nil {
-			return err
-		}
-		logrus.Infof("load image success")
-		return nil
+	if mode == common.ApplyModeLoadImage {
+		return loadToRegistry(infraDriver, distributor)
 	}
 
 	installer := clusterruntime.NewAppInstaller(infraDriver, distributor, extension)

cmd/sealer/cmd/cluster/run.go

@@ -16,6 +16,7 @@ package cluster
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 
@@ -123,7 +124,7 @@ func NewRunCmd() *cobra.Command {
 				return err
 			}
 
-			return createNewCluster(cluster.Spec.Image, infraDriver, imageEngine, cf, applyMode)
+			return createNewCluster(infraDriver, imageEngine, cf, applyMode)
 		},
 	}
 	runFlags = &types.Flags{}
@@ -151,9 +152,10 @@ func NewRunCmd() *cobra.Command {
 	return runCmd
 }
 
-func createNewCluster(clusterImageName string, infraDriver infradriver.InfraDriver, imageEngine imageengine.Interface, cf clusterfile.Interface, createMode string) error {
+func createNewCluster(infraDriver infradriver.InfraDriver, imageEngine imageengine.Interface, cf clusterfile.Interface, mode string) error {
 	var (
-		clusterHosts = infraDriver.GetHostIPList()
+		clusterHosts     = infraDriver.GetHostIPList()
+		clusterImageName = infraDriver.GetClusterImageName()
 	)
 
 	clusterHostsPlatform, err := infraDriver.GetHostsPlatform(clusterHosts)
@@ -183,15 +185,10 @@ func createNewCluster(clusterImageName string, infraDriver infradriver.InfraDriv
 		return err
 	}
 
-	if createMode == common.ApplyModeLoadImage {
-		logrus.Infof("start to apply with mode(%s)", createMode)
-		reg := infraDriver.GetClusterRegistryConfig()
-		if err = distributor.DistributeRegistry(reg.LocalRegistry.DeployHosts, infraDriver.GetClusterRootfsPath()); err != nil {
-			return err
-		}
-		logrus.Infof("load image success")
-		return nil
+	if mode == common.ApplyModeLoadImage {
+		return loadToRegistry(infraDriver, distributor)
 	}
+
 	plugins, err := loadPluginsFromImage(imageMountInfo)
 	if err != nil {
 		return err
@@ -248,3 +245,30 @@ func loadPluginsFromImage(imageMountInfo []imagedistributor.ClusterImageMountInf
 
 	return plugins, nil
 }
+
+// loadToRegistry just load container image to local registry
+func loadToRegistry(infraDriver infradriver.InfraDriver, distributor imagedistributor.Distributor) error {
+	regConfig := infraDriver.GetClusterRegistryConfig()
+	// todo only support load image to local registry at present
+	if regConfig.LocalRegistry == nil {
+		return nil
+	}
+
+	deployHosts := infraDriver.GetHostIPListByRole(common.MASTER)
+	if len(deployHosts) < 1 {
+		return fmt.Errorf("local registry host can not be nil")
+	}
+	master0 := deployHosts[0]
+
+	logrus.Infof("start to apply with mode(%s)", common.ApplyModeLoadImage)
+	if !regConfig.LocalRegistry.HaMode {
+		deployHosts = []net.IP{master0}
+	}
+
+	if err := distributor.DistributeRegistry(deployHosts, filepath.Join(infraDriver.GetClusterRootfsPath(), "registry")); err != nil {
+		return err
+	}
+
+	logrus.Infof("load image success")
+	return nil
+}

cmd/sealer/cmd/utils/cluster.go

@@ -165,7 +165,7 @@ func GetCurrentCluster(client *k8s.Client) (*v2.Cluster, error) {
 		if addr == nil {
 			continue
 		}
-		if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
+		if _, ok := node.Labels[common.MasterRoleLabel]; ok {
 			masterIPList = append(masterIPList, addr)
 			continue
 		}
@@ -180,7 +180,19 @@ func getNodeAddress(node corev1.Node) net.IP {
 	if len(node.Status.Addresses) < 1 {
 		return nil
 	}
-	return net.ParseIP(node.Status.Addresses[0].Address)
+
+	var IP string
+	for _, address := range node.Status.Addresses {
+		if address.Type == "InternalIP" {
+			IP = address.Address
+		}
+	}
+
+	if IP == "" {
+		IP = node.Status.Addresses[0].Address
+	}
+
+	return net.ParseIP(IP)
 }
 
 func GetClusterClient() *k8s.Client {

common/common.go

@@ -73,6 +73,10 @@ const (
 	EnvIPv6DualStack              = "IPv6DualStack"
 )
 
+const (
+	MasterRoleLabel = "node-role.kubernetes.io/master"
+)
+
 const (
 	ApplyModeApply     = "apply"
 	ApplyModeLoadImage = "loadImage"
@@ -127,6 +131,13 @@ const (
 	DefaultRegistryHtPasswdFile = "registry_htpasswd"
 )
 
+// default cluster runtime configuration
+const (
+	DefaultVIP             = "10.103.97.2"
+	DefaultVIPForIPv6      = "1248:4003:10bb:6a01:83b9:6360:c66d:0002"
+	DefaultAPIserverDomain = "apiserver.cluster.local"
+)
+
 const (
 	JoinSubCmd   = "join"
 	DeleteSubCmd = "delete"

pkg/cluster-runtime/apps.go

@@ -48,27 +48,37 @@ func NewAppInstaller(infraDriver infradriver.InfraDriver, distributor imagedistr
 }
 
 func (i *AppInstaller) Install(master0 net.IP, cmds []string) error {
+	masters := i.infraDriver.GetHostIPListByRole(common.MASTER)
+	regConfig := i.infraDriver.GetClusterRegistryConfig()
 	// distribute rootfs
 	if err := i.distributor.Distribute([]net.IP{master0}, i.infraDriver.GetClusterRootfsPath()); err != nil {
 		return err
 	}
 
-	registryConfigurator, err := registry.NewConfigurator(containerruntime.Info{}, i.infraDriver, i.distributor)
-	if err != nil {
-		return err
-	}
+	//if we use local registry service, load container image to registry
+	if regConfig.LocalRegistry != nil {
+		deployHosts := masters
+		if !regConfig.LocalRegistry.HaMode {
+			deployHosts = []net.IP{masters[0]}
+		}
 
-	registryDriver, err := registryConfigurator.GetDriver()
-	if err != nil {
-		return err
-	}
+		registryConfigurator, err := registry.NewConfigurator(deployHosts, containerruntime.Info{}, regConfig, i.infraDriver, i.distributor)
+		if err != nil {
+			return err
+		}
 
-	err = registryDriver.UploadContainerImages2Registry()
-	if err != nil {
-		return err
+		registryDriver, err := registryConfigurator.GetDriver()
+		if err != nil {
+			return err
+		}
+
+		err = registryDriver.UploadContainerImages2Registry()
+		if err != nil {
+			return err
+		}
 	}
 
-	if err = i.LaunchClusterImage(master0, cmds); err != nil {
+	if err := i.LaunchClusterImage(master0, cmds); err != nil {
 		return err
 	}