cli/cmd/cluster: ensure controlplane charts are working before upgrade

This commit modifies logic of 'lokoctl cluster apply' to run equivalent
of 'helm rollback <last release name>' before running 'terraform apply'
to ensure that all controlplane charts are up to date with existing
configuration and that they are healthy before we proceed with upgrades.

This is a safety measure to avoid trying to upgrade the cluster, which
is not healthy, which can only make situation worse.

If controlplane chart is not installed, we attempt to re-install it from
cluster assets directory. If chart is not found there, we bail, as we
have no way of knowing if binary we run right now is the same binary
which was previously used to deploy a cluster and we don't want to
trigger cluster upgrade until we make sure that existing cluster is
healthy.

In such case, user should clone 'lokomotive' repository, copy the right
version of the Helm chart into their assets directory and then re-run
'lokoctl cluster apply'. This will change when we decouple Lokomotive
release from 'lokoctl' binary, as then we will have access to the right
Helm charts.

Signed-off-by: Mateusz Gozdek <[email protected]>

cli/cmd/cluster-apply.go

@@ -23,9 +23,10 @@ import (
 )
 
 var (
-	verbose         bool
-	skipComponents  bool
-	upgradeKubelets bool
+	verbose                  bool
+	skipComponents           bool
+	skipPreUpdateHealthCheck bool
+	upgradeKubelets          bool
 )
 
 var clusterApplyCmd = &cobra.Command{
@@ -43,6 +44,10 @@ func init() {
 	pf.BoolVarP(&confirm, "confirm", "", false, "Upgrade cluster without asking for confirmation")
 	pf.BoolVarP(&verbose, "verbose", "v", false, "Show output from Terraform")
 	pf.BoolVarP(&skipComponents, "skip-components", "", false, "Skip applying component configuration")
+
+	pf.BoolVarP(&skipPreUpdateHealthCheck, "skip-pre-update-health-check", "", false,
+		"Skip ensuring that cluster is healthy before updating (not recommended)")
+
 	pf.BoolVarP(&upgradeKubelets, "upgrade-kubelets", "", false, "Experimentally upgrade self-hosted kubelets")
 }
 
@@ -53,12 +58,13 @@ func runClusterApply(cmd *cobra.Command, args []string) {
 	})
 
 	options := cluster.ApplyOptions{
-		Confirm:         confirm,
-		UpgradeKubelets: upgradeKubelets,
-		SkipComponents:  skipComponents,
-		Verbose:         verbose,
-		ConfigPath:      viper.GetString("lokocfg"),
-		ValuesPath:      viper.GetString("lokocfg-vars"),
+		Confirm:                  confirm,
+		UpgradeKubelets:          upgradeKubelets,
+		SkipComponents:           skipComponents,
+		SkipPreUpdateHealthCheck: skipPreUpdateHealthCheck,
+		Verbose:                  verbose,
+		ConfigPath:               viper.GetString("lokocfg"),
+		ValuesPath:               viper.GetString("lokocfg-vars"),
 	}
 
 	if err := cluster.Apply(contextLogger, options); err != nil {

cli/cmd/cluster/apply.go

@@ -27,12 +27,13 @@ import (
 
 // ApplyOptions defines how cluster apply operation will behave.
 type ApplyOptions struct {
-	Confirm         bool
-	UpgradeKubelets bool
-	SkipComponents  bool
-	Verbose         bool
-	ConfigPath      string
-	ValuesPath      string
+	Confirm                  bool
+	UpgradeKubelets          bool
+	SkipComponents           bool
+	SkipPreUpdateHealthCheck bool
+	Verbose                  bool
+	ConfigPath               string
+	ValuesPath               string
 }
 
 // Apply applies cluster configuration together with components.
@@ -55,9 +56,27 @@ func Apply(contextLogger *log.Entry, options ApplyOptions) error {
 		return fmt.Errorf("checking if cluster exists: %w", err)
 	}
 
-	// Unpack controlplane charts before we return potential errors to the user early.
-	if err := c.unpackControlplaneCharts(); err != nil {
-		return fmt.Errorf("unpacking controlplane assets: %w", err)
+	// Prepare for getting kubeconfig.
+	kg := kubeconfigGetter{
+		platformRequired: true,
+	}
+
+	var kubeconfig []byte
+
+	// Prepare controlplane updater.
+	cu := controlplaneUpdater{
+		kubeconfig:    kubeconfig,
+		assetDir:      c.assetDir,
+		contextLogger: *contextLogger,
+		ex:            c.terraformExecutor,
+	}
+
+	charts := platform.CommonControlPlaneCharts(options.UpgradeKubelets)
+
+	if !exists {
+		if err := c.unpackControlplaneCharts(); err != nil {
+			return fmt.Errorf("unpacking controlplane assets: %w", err)
+		}
 	}
 
 	if exists && !options.Confirm {
@@ -73,17 +92,28 @@ func Apply(contextLogger *log.Entry, options ApplyOptions) error {
 		}
 	}
 
+	if exists && !options.SkipPreUpdateHealthCheck {
+		var err error
+
+		cu.kubeconfig, err = kg.getKubeconfig(contextLogger, c.lokomotiveConfig)
+		if err != nil {
+			return fmt.Errorf("getting kubeconfig: %v", err)
+		}
+
+		for _, c := range charts {
+			if err := cu.ensureComponent(c.Name, c.Namespace); err != nil {
+				return fmt.Errorf("ensuring controlplane component %q: %w", c.Name, err)
+			}
+		}
+	}
+
 	if err := c.platform.Apply(&c.terraformExecutor); err != nil {
 		return fmt.Errorf("applying platform: %v", err)
 	}
 
 	fmt.Printf("\nYour configurations are stored in %s\n", c.assetDir)
 
-	kg := kubeconfigGetter{
-		platformRequired: true,
-	}
-
-	kubeconfig, err := kg.getKubeconfig(contextLogger, c.lokomotiveConfig)
+	kubeconfig, err = kg.getKubeconfig(contextLogger, c.lokomotiveConfig)
 	if err != nil {
 		return fmt.Errorf("getting kubeconfig: %v", err)
 	}
@@ -109,7 +139,9 @@ func Apply(contextLogger *log.Entry, options ApplyOptions) error {
 			ex:            c.terraformExecutor,
 		}
 
-		charts := platform.CommonControlPlaneCharts(options.UpgradeKubelets)
+		if err := c.unpackControlplaneCharts(); err != nil {
+			return fmt.Errorf("unpacking controlplane assets: %w", err)
+		}
 
 		for _, c := range charts {
 			if err := cu.upgradeComponent(c.Name, c.Namespace); err != nil {

docs/cli/lokoctl_cluster_apply.md

@@ -18,11 +18,12 @@ lokoctl cluster apply [flags]
 ### Options
 
 ```
-      --confirm            Upgrade cluster without asking for confirmation
-  -h, --help               help for apply
-      --skip-components    Skip applying component configuration
-      --upgrade-kubelets   Experimentally upgrade self-hosted kubelets
-  -v, --verbose            Show output from Terraform
+      --confirm                        Upgrade cluster without asking for confirmation
+  -h, --help                           help for apply
+      --skip-components                Skip applying component configuration
+      --skip-pre-update-health-check   Skip ensuring that cluster is healthy before updating (not recommended)
+      --upgrade-kubelets               Experimentally upgrade self-hosted kubelets
+  -v, --verbose                        Show output from Terraform
 ```
 
 ### Options inherited from parent commands