From 466b4658b6e5e9ff095a56159a0b5ad8b49bc285 Mon Sep 17 00:00:00 2001 From: Mateusz Gozdek Date: Fri, 13 Nov 2020 16:31:02 +0100 Subject: [PATCH] cli/cmd/cluster: ensure controlplane charts are working before upgrade This commit modifies logic of 'lokoctl cluster apply' to run equivalent of 'helm rollback ' before running 'terraform apply' to ensure that all controlplane charts are up to date with existing configuration and that they are healthy before we proceed with upgrades. This is a safety measure to avoid trying to upgrade the cluster, which is not healthy, which can only make situation worse. If controlplane chart is not installed, we attempt to re-install it from cluster assets directory. If chart is not found there, we bail, as we have no way of knowing if binary we run right now is the same binary which was previously used to deploy a cluster and we don't want to trigger cluster upgrade until we make sure that existing cluster is healthy. In such case, user should clone 'lokomotive' repository, copy the right version of the Helm chart into their assets directory and then re-run 'lokoctl cluster apply'. This will change when we decouple Lokomotive release from 'lokoctl' binary, as then we will have access to the right Helm charts. Signed-off-by: Mateusz Gozdek --- cli/cmd/cluster-apply.go | 24 +++++++----- cli/cmd/cluster/apply.go | 62 +++++++++++++++++++++++-------- docs/cli/lokoctl_cluster_apply.md | 11 +++--- 3 files changed, 68 insertions(+), 29 deletions(-) diff --git a/cli/cmd/cluster-apply.go b/cli/cmd/cluster-apply.go index 26239420d..c34849082 100644 --- a/cli/cmd/cluster-apply.go +++ b/cli/cmd/cluster-apply.go @@ -23,9 +23,10 @@ import ( ) var ( - verbose bool - skipComponents bool - upgradeKubelets bool + verbose bool + skipComponents bool + skipPreUpdateHealthCheck bool + upgradeKubelets bool ) var clusterApplyCmd = &cobra.Command{ @@ -43,6 +44,10 @@ func init() { pf.BoolVarP(&confirm, "confirm", "", false, "Upgrade cluster without asking for confirmation") pf.BoolVarP(&verbose, "verbose", "v", false, "Show output from Terraform") pf.BoolVarP(&skipComponents, "skip-components", "", false, "Skip applying component configuration") + + pf.BoolVarP(&skipPreUpdateHealthCheck, "skip-pre-update-health-check", "", false, + "Skip ensuring that cluster is healthy before updating (not recommended)") + pf.BoolVarP(&upgradeKubelets, "upgrade-kubelets", "", false, "Experimentally upgrade self-hosted kubelets") } @@ -53,12 +58,13 @@ func runClusterApply(cmd *cobra.Command, args []string) { }) options := cluster.ApplyOptions{ - Confirm: confirm, - UpgradeKubelets: upgradeKubelets, - SkipComponents: skipComponents, - Verbose: verbose, - ConfigPath: viper.GetString("lokocfg"), - ValuesPath: viper.GetString("lokocfg-vars"), + Confirm: confirm, + UpgradeKubelets: upgradeKubelets, + SkipComponents: skipComponents, + SkipPreUpdateHealthCheck: skipPreUpdateHealthCheck, + Verbose: verbose, + ConfigPath: viper.GetString("lokocfg"), + ValuesPath: viper.GetString("lokocfg-vars"), } if err := cluster.Apply(contextLogger, options); err != nil { diff --git a/cli/cmd/cluster/apply.go b/cli/cmd/cluster/apply.go index a5cc5ffe3..7ed8f6110 100644 --- a/cli/cmd/cluster/apply.go +++ b/cli/cmd/cluster/apply.go @@ -27,12 +27,13 @@ import ( // ApplyOptions defines how cluster apply operation will behave. type ApplyOptions struct { - Confirm bool - UpgradeKubelets bool - SkipComponents bool - Verbose bool - ConfigPath string - ValuesPath string + Confirm bool + UpgradeKubelets bool + SkipComponents bool + SkipPreUpdateHealthCheck bool + Verbose bool + ConfigPath string + ValuesPath string } // Apply applies cluster configuration together with components. @@ -55,9 +56,27 @@ func Apply(contextLogger *log.Entry, options ApplyOptions) error { return fmt.Errorf("checking if cluster exists: %w", err) } - // Unpack controlplane charts before we return potential errors to the user early. - if err := c.unpackControlplaneCharts(); err != nil { - return fmt.Errorf("unpacking controlplane assets: %w", err) + // Prepare for getting kubeconfig. + kg := kubeconfigGetter{ + platformRequired: true, + } + + var kubeconfig []byte + + // Prepare controlplane updater. + cu := controlplaneUpdater{ + kubeconfig: kubeconfig, + assetDir: c.assetDir, + contextLogger: *contextLogger, + ex: c.terraformExecutor, + } + + charts := platform.CommonControlPlaneCharts(options.UpgradeKubelets) + + if !exists { + if err := c.unpackControlplaneCharts(); err != nil { + return fmt.Errorf("unpacking controlplane assets: %w", err) + } } if exists && !options.Confirm { @@ -73,17 +92,28 @@ func Apply(contextLogger *log.Entry, options ApplyOptions) error { } } + if exists && !options.SkipPreUpdateHealthCheck { + var err error + + cu.kubeconfig, err = kg.getKubeconfig(contextLogger, c.lokomotiveConfig) + if err != nil { + return fmt.Errorf("getting kubeconfig: %v", err) + } + + for _, c := range charts { + if err := cu.ensureComponent(c.Name, c.Namespace); err != nil { + return fmt.Errorf("ensuring controlplane component %q: %w", c.Name, err) + } + } + } + if err := c.platform.Apply(&c.terraformExecutor); err != nil { return fmt.Errorf("applying platform: %v", err) } fmt.Printf("\nYour configurations are stored in %s\n", c.assetDir) - kg := kubeconfigGetter{ - platformRequired: true, - } - - kubeconfig, err := kg.getKubeconfig(contextLogger, c.lokomotiveConfig) + kubeconfig, err = kg.getKubeconfig(contextLogger, c.lokomotiveConfig) if err != nil { return fmt.Errorf("getting kubeconfig: %v", err) } @@ -109,7 +139,9 @@ func Apply(contextLogger *log.Entry, options ApplyOptions) error { ex: c.terraformExecutor, } - charts := platform.CommonControlPlaneCharts(options.UpgradeKubelets) + if err := c.unpackControlplaneCharts(); err != nil { + return fmt.Errorf("unpacking controlplane assets: %w", err) + } for _, c := range charts { if err := cu.upgradeComponent(c.Name, c.Namespace); err != nil { diff --git a/docs/cli/lokoctl_cluster_apply.md b/docs/cli/lokoctl_cluster_apply.md index 8d0917511..19bce7ebe 100644 --- a/docs/cli/lokoctl_cluster_apply.md +++ b/docs/cli/lokoctl_cluster_apply.md @@ -18,11 +18,12 @@ lokoctl cluster apply [flags] ### Options ``` - --confirm Upgrade cluster without asking for confirmation - -h, --help help for apply - --skip-components Skip applying component configuration - --upgrade-kubelets Experimentally upgrade self-hosted kubelets - -v, --verbose Show output from Terraform + --confirm Upgrade cluster without asking for confirmation + -h, --help help for apply + --skip-components Skip applying component configuration + --skip-pre-update-health-check Skip ensuring that cluster is healthy before updating (not recommended) + --upgrade-kubelets Experimentally upgrade self-hosted kubelets + -v, --verbose Show output from Terraform ``` ### Options inherited from parent commands