feat(*): Ports --cleanup-on-fail to v3

This ports the functionality of cleanup on fail to v3 as introduced in #4871. This has been tested manually
and would be a good candidate for a new acceptance test.

Signed-off-by: Taylor Thomas <taylor.thomas@microsoft.com>

cmd/helm/rollback.go

@@ -71,6 +71,7 @@ func newRollbackCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
 	f.BoolVar(&client.DisableHooks, "no-hooks", false, "prevent hooks from running during rollback")
 	f.DurationVar(&client.Timeout, "timeout", 300*time.Second, "time to wait for any individual Kubernetes operation (like Jobs for hooks)")
 	f.BoolVar(&client.Wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, or ReplicaSet are in a ready state before marking the release as successful. It will wait for as long as --timeout")
+	f.BoolVar(&client.CleanupOnFail, "cleanup-on-fail", false, "allow deletion of new resources created in this rollback when rollback fails")
 
 	return cmd
 }

cmd/helm/upgrade.go

@@ -155,6 +155,7 @@ func newUpgradeCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
 	f.BoolVar(&client.Wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, or ReplicaSet are in a ready state before marking the release as successful. It will wait for as long as --timeout")
 	f.BoolVar(&client.Atomic, "atomic", false, "if set, upgrade process rolls back changes made in case of failed upgrade. The --wait flag will be set automatically if --atomic is used")
 	f.IntVar(&client.MaxHistory, "history-max", 10, "limit the maximum number of revisions saved per release. Use 0 for no limit.")
+	f.BoolVar(&client.CleanupOnFail, "cleanup-on-fail", false, "allow deletion of new resources created in this upgrade when upgrade fails")
 	addChartPathOptionsFlags(f, &client.ChartPathOptions)
 	addValueOptionsFlags(f, valueOpts)
 

pkg/action/rollback.go

@@ -19,6 +19,7 @@ package action
 import (
 	"bytes"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -32,13 +33,14 @@ import (
 type Rollback struct {
 	cfg *Configuration
 
-	Version      int
-	Timeout      time.Duration
-	Wait         bool
-	DisableHooks bool
-	DryRun       bool
-	Recreate     bool // will (if true) recreate pods after a rollback.
-	Force        bool // will (if true) force resource upgrade through uninstall/recreate if needed
+	Version       int
+	Timeout       time.Duration
+	Wait          bool
+	DisableHooks  bool
+	DryRun        bool
+	Recreate      bool // will (if true) recreate pods after a rollback.
+	Force         bool // will (if true) force resource upgrade through uninstall/recreate if needed
+	CleanupOnFail bool
 }
 
 // NewRollback creates a new Rollback object with the given configuration.
@@ -66,6 +68,7 @@ func (r *Rollback) Run(name string) error {
 			return err
 		}
 	}
+
 	r.cfg.Log("performing rollback of %s", name)
 	if _, err := r.performRollback(currentRelease, targetRelease); err != nil {
 		return err
@@ -165,6 +168,18 @@ func (r *Rollback) performRollback(currentRelease, targetRelease *release.Releas
 		targetRelease.Info.Description = msg
 		r.cfg.recordRelease(currentRelease)
 		r.cfg.recordRelease(targetRelease)
+		if r.CleanupOnFail {
+			r.cfg.Log("Cleanup on fail set, cleaning up %d resources", len(results.Created))
+			_, errs := r.cfg.KubeClient.Delete(results.Created)
+			if errs != nil {
+				var errorList []string
+				for _, e := range errs {
+					errorList = append(errorList, e.Error())
+				}
+				return targetRelease, errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s", strings.Join(errorList, ", ")), "an error occurred while cleaning up resources. original rollback error: %s", err)
+			}
+			r.cfg.Log("Resource cleanup complete")
+		}
 		return targetRelease, err
 	}
 

pkg/action/upgrade.go

@@ -19,6 +19,7 @@ package action
 import (
 	"bytes"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -52,8 +53,9 @@ type Upgrade struct {
 	// Recreate will (if true) recreate pods after a rollback.
 	Recreate bool
 	// MaxHistory limits the maximum number of revisions saved per release
-	MaxHistory int
-	Atomic     bool
+	MaxHistory    int
+	Atomic        bool
+	CleanupOnFail bool
 }
 
 // NewUpgrade creates a new Upgrade object with the given configuration.
@@ -210,7 +212,7 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea
 	// pre-upgrade hooks
 	if !u.DisableHooks {
 		if err := u.cfg.execHook(upgradedRelease, release.HookPreUpgrade, u.Timeout); err != nil {
-			return u.failRelease(upgradedRelease, fmt.Errorf("pre-upgrade hooks failed: %s", err))
+			return u.failRelease(upgradedRelease, kube.ResourceList{}, fmt.Errorf("pre-upgrade hooks failed: %s", err))
 		}
 	} else {
 		u.cfg.Log("upgrade hooks disabled for %s", upgradedRelease.Name)
@@ -219,7 +221,7 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea
 	results, err := u.cfg.KubeClient.Update(current, target, u.Force)
 	if err != nil {
 		u.cfg.recordRelease(originalRelease)
-		return u.failRelease(upgradedRelease, err)
+		return u.failRelease(upgradedRelease, results.Created, err)
 	}
 
 	if u.Recreate {
@@ -235,14 +237,14 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea
 	if u.Wait {
 		if err := u.cfg.KubeClient.Wait(target, u.Timeout); err != nil {
 			u.cfg.recordRelease(originalRelease)
-			return u.failRelease(upgradedRelease, err)
+			return u.failRelease(upgradedRelease, results.Created, err)
 		}
 	}
 
 	// post-upgrade hooks
 	if !u.DisableHooks {
 		if err := u.cfg.execHook(upgradedRelease, release.HookPostUpgrade, u.Timeout); err != nil {
-			return u.failRelease(upgradedRelease, fmt.Errorf("post-upgrade hooks failed: %s", err))
+			return u.failRelease(upgradedRelease, results.Created, fmt.Errorf("post-upgrade hooks failed: %s", err))
 		}
 	}
 
@@ -255,13 +257,25 @@ func (u *Upgrade) performUpgrade(originalRelease, upgradedRelease *release.Relea
 	return upgradedRelease, nil
 }
 
-func (u *Upgrade) failRelease(rel *release.Release, err error) (*release.Release, error) {
+func (u *Upgrade) failRelease(rel *release.Release, created kube.ResourceList, err error) (*release.Release, error) {
 	msg := fmt.Sprintf("Upgrade %q failed: %s", rel.Name, err)
 	u.cfg.Log("warning: %s", msg)
 
 	rel.Info.Status = release.StatusFailed
 	rel.Info.Description = msg
 	u.cfg.recordRelease(rel)
+	if u.CleanupOnFail {
+		u.cfg.Log("Cleanup on fail set, cleaning up %d resources", len(created))
+		_, errs := u.cfg.KubeClient.Delete(created)
+		if errs != nil {
+			var errorList []string
+			for _, e := range errs {
+				errorList = append(errorList, e.Error())
+			}
+			return rel, errors.Wrapf(fmt.Errorf("unable to cleanup resources: %s", strings.Join(errorList, ", ")), "an error occurred while cleaning up resources. original upgrade error: %s", err)
+		}
+		u.cfg.Log("Resource cleanup complete")
+	}
 	if u.Atomic {
 		u.cfg.Log("Upgrade failed and atomic is set, rolling back to last successful release")
 

pkg/kube/client.go

@@ -129,10 +129,13 @@ func (c *Client) Build(reader io.Reader) (ResourceList, error) {
 	return result, scrubValidationError(err)
 }
 
-// Update reads in the current configuration and a target configuration from io.reader
-// and creates resources that don't already exists, updates resources that have been modified
-// in the target configuration and deletes resources from the current configuration that are
-// not present in the target configuration.
+// Update takes the current list of objects and target list of objects and
+// creates resources that don't already exists, updates resources that have been
+// modified in the target configuration, and deletes resources from the current
+// configuration that are not present in the target configuration. If an error
+// occurs, a Result will still be returned with the error, containing all
+// resource updates, creations, and deletions that were attempted. These can be
+// used for cleanup or other logging purposes.
 func (c *Client) Update(original, target ResourceList, force bool) (*Result, error) {
 	updateErrors := []string{}
 	res := &Result{}
@@ -149,14 +152,14 @@ func (c *Client) Update(original, target ResourceList, force bool) (*Result, err
 				return errors.Wrap(err, "could not get information about the resource")
 			}
 
+			// Append the created resource to the results, even if something fails
+			res.Created = append(res.Created, info)
+
 			// Since the resource does not exist, create it.
 			if err := createResource(info); err != nil {
 				return errors.Wrap(err, "failed to create resource")
 			}
 
-			// Append the created resource to the results
-			res.Created = append(res.Created, info)
-
 			kind := info.Mapping.GroupVersionKind.Kind
 			c.Log("Created a new %s called %q\n", kind, info.Name)
 			return nil
@@ -180,18 +183,17 @@ func (c *Client) Update(original, target ResourceList, force bool) (*Result, err
 
 	switch {
 	case err != nil:
-		return nil, err
+		return res, err
 	case len(updateErrors) != 0:
-		return nil, errors.Errorf(strings.Join(updateErrors, " && "))
+		return res, errors.Errorf(strings.Join(updateErrors, " && "))
 	}
 
 	for _, info := range original.Difference(target) {
 		c.Log("Deleting %q in %s...", info.Name, info.Namespace)
+		res.Deleted = append(res.Deleted, info)
 		if err := deleteResource(info); err != nil {
 			c.Log("Failed to delete %q, err: %s", info.Name, err)
-		} else {
-			// Only append ones we succeeded in deleting
-			res.Deleted = append(res.Deleted, info)
+			return res, errors.Wrapf(err, "Failed to delete %q", info.Name)
 		}
 	}
 	return res, nil

pkg/kube/fake/fake.go

@@ -77,7 +77,7 @@ func (f *FailingKubeClient) WatchUntilReady(resources kube.ResourceList, d time.
 // Update returns the configured error if set or prints
 func (f *FailingKubeClient) Update(r, modified kube.ResourceList, ignoreMe bool) (*kube.Result, error) {
 	if f.UpdateError != nil {
-		return nil, f.UpdateError
+		return &kube.Result{}, f.UpdateError
 	}
 	return f.PrintingKubeClient.Update(r, modified, ignoreMe)
 }