diff --git a/pkg/controllers/machine/terminator/terminator.go b/pkg/controllers/machine/terminator/terminator.go index e8aa87a7da..1e37e25b65 100644 --- a/pkg/controllers/machine/terminator/terminator.go +++ b/pkg/controllers/machine/terminator/terminator.go @@ -111,7 +111,8 @@ func (t *Terminator) TerminateNode(ctx context.Context, node *v1.Node) error { if err := t.kubeClient.Patch(ctx, node, client.MergeFrom(stored)); err != nil { return err } - terminationSummary.Observe(time.Since(node.DeletionTimestamp.Time).Seconds()) + // api-server may give back a node after the patch without a deletionTimestamp if we use node.DeletionTimestamp.Time + terminationSummary.Observe(time.Since(stored.DeletionTimestamp.Time).Seconds()) } return nil } diff --git a/pkg/controllers/termination/controller.go b/pkg/controllers/termination/controller.go index cc5fe75ed9..dcece57c80 100644 --- a/pkg/controllers/termination/controller.go +++ b/pkg/controllers/termination/controller.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" @@ -66,17 +67,17 @@ func (c *Controller) Finalize(ctx context.Context, node *v1.Node) (reconcile.Res return reconcile.Result{}, nil } if err := c.terminator.Cordon(ctx, node); err != nil { - return reconcile.Result{}, fmt.Errorf("cordoning node, %w", err) + return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("cordoning node, %w", err)) } if err := c.terminator.Drain(ctx, node); err != nil { if terminator.IsNodeDrainError(err) { c.recorder.Publish(events.NodeFailedToDrain(node, err)) return reconcile.Result{Requeue: true}, nil } - return reconcile.Result{}, fmt.Errorf("draining node, %w", err) + return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("draining node, %w", err)) } if err := c.terminator.TerminateNode(ctx, node); err != nil { - return reconcile.Result{}, fmt.Errorf("terminating node, %w", err) + return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("terminating node, %w", err)) } return reconcile.Result{}, nil } @@ -85,6 +86,7 @@ func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontrolle return corecontroller.Adapt(controllerruntime. NewControllerManagedBy(m). For(&v1.Node{}). + WithEventFilter(predicate.GenerationChangedPredicate{}). // Ignore updates that are generated by this controller WithOptions( controller.Options{ RateLimiter: workqueue.NewMaxOfRateLimiter(