Add --force to upgrade and rollback

_proto/hapi/rudder/rudder.proto

@@ -91,6 +91,7 @@ message UpgradeReleaseRequest{
         int64 Timeout = 3;
         bool Wait = 4;
         bool Recreate = 5;
+        bool Force = 6;
 }
 message UpgradeReleaseResponse{
 	hapi.release.Release release = 1;
@@ -103,6 +104,7 @@ message RollbackReleaseRequest{
         int64 Timeout = 3;
         bool Wait = 4;
         bool Recreate = 5;
+        bool Force = 6;
 }
 message RollbackReleaseResponse{
 	hapi.release.Release release = 1;

_proto/hapi/services/tiller.proto

@@ -207,6 +207,8 @@ message UpdateReleaseRequest {
 	// ReuseValues will cause Tiller to reuse the values from the last release.
 	// This is ignored if reset_values is set.
 	bool reuse_values = 10;
+	// Force resource update through delete/recreate if needed.
+	bool force = 11;
 }
 
 // UpdateReleaseResponse is the response to an update request.
@@ -230,6 +232,8 @@ message RollbackReleaseRequest {
 	// wait, if true, will wait until all Pods, PVCs, and Services are in a ready state
 	// before marking the release as successful. It will wait for as long as timeout
 	bool wait = 7;
+	// Force resource update through delete/recreate if needed.
+	bool force = 8;
 }
 
 // RollbackReleaseResponse is the response to an update request.

cmd/helm/rollback.go

@@ -39,6 +39,7 @@ type rollbackCmd struct {
 	revision     int32
 	dryRun       bool
 	recreate     bool
+	force        bool
 	disableHooks bool
 	out          io.Writer
 	client       helm.Interface
@@ -78,6 +79,7 @@ func newRollbackCmd(c helm.Interface, out io.Writer) *cobra.Command {
 	f := cmd.Flags()
 	f.BoolVar(&rollback.dryRun, "dry-run", false, "simulate a rollback")
 	f.BoolVar(&rollback.recreate, "recreate-pods", false, "performs pods restart for the resource if applicable")
+	f.BoolVar(&rollback.force, "force", false, "force resource update through delete/recreate if needed")
 	f.BoolVar(&rollback.disableHooks, "no-hooks", false, "prevent hooks from running during rollback")
 	f.Int64Var(&rollback.timeout, "timeout", 300, "time in seconds to wait for any individual kubernetes operation (like Jobs for hooks)")
 	f.BoolVar(&rollback.wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment are in a ready state before marking the release as successful. It will wait for as long as --timeout")
@@ -90,6 +92,7 @@ func (r *rollbackCmd) run() error {
 		r.name,
 		helm.RollbackDryRun(r.dryRun),
 		helm.RollbackRecreate(r.recreate),
+		helm.RollbackForce(r.force),
 		helm.RollbackDisableHooks(r.disableHooks),
 		helm.RollbackVersion(r.revision),
 		helm.RollbackTimeout(r.timeout),

cmd/helm/upgrade.go

@@ -62,6 +62,7 @@ type upgradeCmd struct {
 	client       helm.Interface
 	dryRun       bool
 	recreate     bool
+	force        bool
 	disableHooks bool
 	valueFiles   valueFiles
 	values       []string
@@ -116,6 +117,7 @@ func newUpgradeCmd(client helm.Interface, out io.Writer) *cobra.Command {
 	f.VarP(&upgrade.valueFiles, "values", "f", "specify values in a YAML file (can specify multiple)")
 	f.BoolVar(&upgrade.dryRun, "dry-run", false, "simulate an upgrade")
 	f.BoolVar(&upgrade.recreate, "recreate-pods", false, "performs pods restart for the resource if applicable")
+	f.BoolVar(&upgrade.force, "force", false, "force resource update through delete/recreate if needed")
 	f.StringArrayVar(&upgrade.values, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	f.BoolVar(&upgrade.disableHooks, "disable-hooks", false, "disable pre/post upgrade hooks. DEPRECATED. Use no-hooks")
 	f.BoolVar(&upgrade.disableHooks, "no-hooks", false, "disable pre/post upgrade hooks")
@@ -198,6 +200,7 @@ func (u *upgradeCmd) run() error {
 		helm.UpdateValueOverrides(rawVals),
 		helm.UpgradeDryRun(u.dryRun),
 		helm.UpgradeRecreate(u.recreate),
+		helm.UpgradeForce(u.force),
 		helm.UpgradeDisableHooks(u.disableHooks),
 		helm.UpgradeTimeout(u.timeout),
 		helm.ResetValues(u.resetValues),

cmd/rudder/rudder.go

@@ -112,7 +112,7 @@ func (r *ReleaseModuleServiceServer) RollbackRelease(ctx context.Context, in *ru
 	grpclog.Print("rollback")
 	c := bytes.NewBufferString(in.Current.Manifest)
 	t := bytes.NewBufferString(in.Target.Manifest)
-	err := kubeClient.Update(in.Target.Namespace, c, t, in.Recreate, in.Timeout, in.Wait)
+	err := kubeClient.Update(in.Target.Namespace, c, t, in.Force, in.Recreate, in.Timeout, in.Wait)
 	return &rudderAPI.RollbackReleaseResponse{}, err
 }
 
@@ -121,7 +121,7 @@ func (r *ReleaseModuleServiceServer) UpgradeRelease(ctx context.Context, in *rud
 	grpclog.Print("upgrade")
 	c := bytes.NewBufferString(in.Current.Manifest)
 	t := bytes.NewBufferString(in.Target.Manifest)
-	err := kubeClient.Update(in.Target.Namespace, c, t, in.Recreate, in.Timeout, in.Wait)
+	err := kubeClient.Update(in.Target.Namespace, c, t, in.Force, in.Recreate, in.Timeout, in.Wait)
 	// upgrade response object should be changed to include status
 	return &rudderAPI.UpgradeReleaseResponse{}, err
 }

pkg/helm/client.go

@@ -159,6 +159,7 @@ func (h *Client) UpdateReleaseFromChart(rlsName string, chart *chart.Chart, opts
 	req.Name = rlsName
 	req.DisableHooks = h.opts.disableHooks
 	req.Recreate = h.opts.recreate
+	req.Force = h.opts.force
 	req.ResetValues = h.opts.resetValues
 	req.ReuseValues = h.opts.reuseValues
 	ctx := NewContext()
@@ -202,6 +203,8 @@ func (h *Client) RollbackRelease(rlsName string, opts ...RollbackOption) (*rls.R
 		opt(&h.opts)
 	}
 	req := &h.opts.rollbackReq
+	req.Recreate = h.opts.recreate
+	req.Force = h.opts.force
 	req.DisableHooks = h.opts.disableHooks
 	req.DryRun = h.opts.dryRun
 	req.Name = rlsName

pkg/helm/option.go

@@ -46,6 +46,8 @@ type options struct {
 	reuseName bool
 	// if set, performs pod restart during upgrade/rollback
 	recreate bool
+	// if set, force resource update through delete/recreate if needed
+	force bool
 	// if set, skip running hooks
 	disableHooks bool
 	// name of release
@@ -311,6 +313,13 @@ func RollbackRecreate(recreate bool) RollbackOption {
 	}
 }
 
+// RollbackForce will (if true) force resource update through delete/recreate if needed
+func RollbackForce(force bool) RollbackOption {
+	return func(opts *options) {
+		opts.force = force
+	}
+}
+
 // RollbackVersion sets the version of the release to deploy.
 func RollbackVersion(ver int32) RollbackOption {
 	return func(opts *options) {
@@ -353,6 +362,13 @@ func UpgradeRecreate(recreate bool) UpdateOption {
 	}
 }
 
+// UpgradeForce will (if true) force resource update through delete/recreate if needed
+func UpgradeForce(force bool) UpdateOption {
+	return func(opts *options) {
+		opts.force = force
+	}
+}
+
 // ContentOption allows setting optional attributes when
 // performing a GetReleaseContent tiller rpc.
 type ContentOption func(*options)

pkg/kube/client.go

@@ -211,7 +211,7 @@ func (c *Client) Get(namespace string, reader io.Reader) (string, error) {
 //  not present in the target configuration
 //
 // Namespace will set the namespaces
-func (c *Client) Update(namespace string, originalReader, targetReader io.Reader, recreate bool, timeout int64, shouldWait bool) error {
+func (c *Client) Update(namespace string, originalReader, targetReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error {
 	original, err := c.BuildUnstructured(namespace, originalReader)
 	if err != nil {
 		return fmt.Errorf("failed decoding reader into objects: %s", err)
@@ -250,7 +250,7 @@ func (c *Client) Update(namespace string, originalReader, targetReader io.Reader
 			return fmt.Errorf("no resource with the name %q found", info.Name)
 		}
 
-		if err := updateResource(c, info, originalInfo.Object, recreate); err != nil {
+		if err := updateResource(c, info, originalInfo.Object, force, recreate); err != nil {
 			c.Log("error updating the resource %q:\n\t %v", info.Name, err)
 			updateErrors = append(updateErrors, err.Error())
 		}
@@ -392,7 +392,7 @@ func createPatch(mapping *meta.RESTMapping, target, current runtime.Object) ([]b
 	}
 }
 
-func updateResource(c *Client, target *resource.Info, currentObj runtime.Object, recreate bool) error {
+func updateResource(c *Client, target *resource.Info, currentObj runtime.Object, force bool, recreate bool) error {
 	patch, patchType, err := createPatch(target.Mapping, target.Object, currentObj)
 	if err != nil {
 		return fmt.Errorf("failed to create patch: %s", err)
@@ -409,12 +409,36 @@ func updateResource(c *Client, target *resource.Info, currentObj runtime.Object,
 
 	// send patch to server
 	helper := resource.NewHelper(target.Client, target.Mapping)
+
 	obj, err := helper.Patch(target.Namespace, target.Name, patchType, patch)
 	if err != nil {
-		return err
-	}
+		kind := target.Mapping.GroupVersionKind.Kind
+		log.Printf("Cannot patch %s: %q (%v)", kind, target.Name, err)
+
+		if force {
+			// Attempt to delete...
+			if err := deleteResource(c, target); err != nil {
+				return err
+			}
+			log.Printf("Deleted %s: %q", kind, target.Name)
 
-	target.Refresh(obj, true)
+			// ... and recreate
+			if err := createResource(target); err != nil {
+				return fmt.Errorf("Failed to recreate resource: %s", err)
+			}
+			log.Printf("Created a new %s called %q\n", kind, target.Name)
+
+			// No need to refresh the target, as we recreated the resource based
+			// on it. In addition, it might not exist yet and a call to `Refresh`
+			// may fail.
+		} else {
+			log.Print("Use --force to force recreation of the resource")
+			return err
+		}
+	} else {
+		// When patch succeeds without needing to recreate, refresh target.
+		target.Refresh(obj, true)
+	}
 
 	if !recreate {
 		return nil

pkg/kube/client_test.go

@@ -193,7 +193,7 @@ func TestUpdate(t *testing.T) {
 	reaper := &fakeReaper{}
 	rf := &fakeReaperFactory{Factory: f, reaper: reaper}
 	c := newTestClient(rf)
-	if err := c.Update(api.NamespaceDefault, objBody(codec, &listA), objBody(codec, &listB), false, 0, false); err != nil {
+	if err := c.Update(api.NamespaceDefault, objBody(codec, &listA), objBody(codec, &listB), false, false, 0, false); err != nil {
 		t.Fatal(err)
 	}
 	// TODO: Find a way to test methods that use Client Set

pkg/tiller/environment/environment.go

@@ -147,7 +147,7 @@ type KubeClient interface {
 	//
 	// reader must contain a YAML stream (one or more YAML documents separated
 	// by "\n---\n").
-	Update(namespace string, originalReader, modifiedReader io.Reader, recreate bool, timeout int64, shouldWait bool) error
+	Update(namespace string, originalReader, modifiedReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error
 
 	Build(namespace string, reader io.Reader) (kube.Result, error)
 	BuildUnstructured(namespace string, reader io.Reader) (kube.Result, error)
@@ -190,7 +190,7 @@ func (p *PrintingKubeClient) WatchUntilReady(ns string, r io.Reader, timeout int
 }
 
 // Update implements KubeClient Update.
-func (p *PrintingKubeClient) Update(ns string, currentReader, modifiedReader io.Reader, recreate bool, timeout int64, shouldWait bool) error {
+func (p *PrintingKubeClient) Update(ns string, currentReader, modifiedReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error {
 	_, err := io.Copy(p.Out, modifiedReader)
 	return err
 }

pkg/tiller/environment/environment_test.go

@@ -48,7 +48,7 @@ func (k *mockKubeClient) Get(ns string, r io.Reader) (string, error) {
 func (k *mockKubeClient) Delete(ns string, r io.Reader) error {
 	return nil
 }
-func (k *mockKubeClient) Update(ns string, currentReader, modifiedReader io.Reader, recreate bool, timeout int64, shouldWait bool) error {
+func (k *mockKubeClient) Update(ns string, currentReader, modifiedReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error {
 	return nil
 }
 func (k *mockKubeClient) WatchUntilReady(ns string, r io.Reader, timeout int64, shouldWait bool) error {

pkg/tiller/release_modules.go

@@ -59,14 +59,14 @@ func (m *LocalReleaseModule) Create(r *release.Release, req *services.InstallRel
 func (m *LocalReleaseModule) Update(current, target *release.Release, req *services.UpdateReleaseRequest, env *environment.Environment) error {
 	c := bytes.NewBufferString(current.Manifest)
 	t := bytes.NewBufferString(target.Manifest)
-	return env.KubeClient.Update(target.Namespace, c, t, req.Recreate, req.Timeout, req.Wait)
+	return env.KubeClient.Update(target.Namespace, c, t, req.Force, req.Recreate, req.Timeout, req.Wait)
 }
 
 // Rollback performs a rollback from current to target release
 func (m *LocalReleaseModule) Rollback(current, target *release.Release, req *services.RollbackReleaseRequest, env *environment.Environment) error {
 	c := bytes.NewBufferString(current.Manifest)
 	t := bytes.NewBufferString(target.Manifest)
-	return env.KubeClient.Update(target.Namespace, c, t, req.Recreate, req.Timeout, req.Wait)
+	return env.KubeClient.Update(target.Namespace, c, t, req.Force, req.Recreate, req.Timeout, req.Wait)
 }
 
 // Status returns kubectl-like formatted status of release objects
@@ -101,6 +101,7 @@ func (m *RemoteReleaseModule) Update(current, target *release.Release, req *serv
 		Recreate: req.Recreate,
 		Timeout:  req.Timeout,
 		Wait:     req.Wait,
+		Force:    req.Force,
 	}
 	_, err := rudder.UpgradeRelease(upgrade)
 	return err

pkg/tiller/release_server_test.go

@@ -307,7 +307,7 @@ type updateFailingKubeClient struct {
 	environment.PrintingKubeClient
 }
 
-func (u *updateFailingKubeClient) Update(namespace string, originalReader, modifiedReader io.Reader, recreate bool, timeout int64, shouldWait bool) error {
+func (u *updateFailingKubeClient) Update(namespace string, originalReader, modifiedReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error {
 	return errors.New("Failed update in kube client")
 }