Fix panic in termination flow

kubernetes-sigs · Feb 6, 2023 · 11460fe · 11460fe
1 parent a6491a7
commit 11460fe
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/pkg/controllers/machine/terminator/terminator.go b/pkg/controllers/machine/terminator/terminator.go
@@ -111,7 +111,8 @@ func (t *Terminator) TerminateNode(ctx context.Context, node *v1.Node) error {
 		if err := t.kubeClient.Patch(ctx, node, client.MergeFrom(stored)); err != nil {
 			return err
 		}
-		terminationSummary.Observe(time.Since(node.DeletionTimestamp.Time).Seconds())
+		// api-server may give back a node after the patch without a deletionTimestamp if we use node.DeletionTimestamp.Time
+		terminationSummary.Observe(time.Since(stored.DeletionTimestamp.Time).Seconds())
 	}
 	return nil
 }

diff --git a/pkg/controllers/termination/controller.go b/pkg/controllers/termination/controller.go
@@ -27,6 +27,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
@@ -66,17 +67,17 @@ func (c *Controller) Finalize(ctx context.Context, node *v1.Node) (reconcile.Res
 		return reconcile.Result{}, nil
 	}
 	if err := c.terminator.Cordon(ctx, node); err != nil {
-		return reconcile.Result{}, fmt.Errorf("cordoning node, %w", err)
+		return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("cordoning node, %w", err))
 	}
 	if err := c.terminator.Drain(ctx, node); err != nil {
 		if terminator.IsNodeDrainError(err) {
 			c.recorder.Publish(events.NodeFailedToDrain(node, err))
 			return reconcile.Result{Requeue: true}, nil
 		}
-		return reconcile.Result{}, fmt.Errorf("draining node, %w", err)
+		return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("draining node, %w", err))
 	}
 	if err := c.terminator.TerminateNode(ctx, node); err != nil {
-		return reconcile.Result{}, fmt.Errorf("terminating node, %w", err)
+		return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("terminating node, %w", err))
 	}
 	return reconcile.Result{}, nil
 }
@@ -85,6 +86,7 @@ func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontrolle
 	return corecontroller.Adapt(controllerruntime.
 		NewControllerManagedBy(m).
 		For(&v1.Node{}).
+		WithEventFilter(predicate.GenerationChangedPredicate{}). // Ignore updates that are generated by this controller
 		WithOptions(
 			controller.Options{
 				RateLimiter: workqueue.NewMaxOfRateLimiter(