Feature: support DeleteOptions when deleting resources in member clusters

pkg/client/generic/genericclient.go

@@ -31,7 +31,7 @@ type Client interface {
 	Create(ctx context.Context, obj runtime.Object) error
 	Get(ctx context.Context, obj runtime.Object, namespace, name string) error
 	Update(ctx context.Context, obj runtime.Object) error
-	Delete(ctx context.Context, obj runtime.Object, namespace, name string) error
+	Delete(ctx context.Context, obj runtime.Object, namespace, name string, opts ...client.DeleteOption) error
 	List(ctx context.Context, obj runtime.Object, namespace string, opts ...client.ListOption) error
 	UpdateStatus(ctx context.Context, obj runtime.Object) error
 }
@@ -71,14 +71,14 @@ func (c *genericClient) Update(ctx context.Context, obj runtime.Object) error {
 	return c.client.Update(ctx, obj)
 }
 
-func (c *genericClient) Delete(ctx context.Context, obj runtime.Object, namespace, name string) error {
+func (c *genericClient) Delete(ctx context.Context, obj runtime.Object, namespace, name string, opts ...client.DeleteOption) error {
 	accessor, err := meta.Accessor(obj)
 	if err != nil {
 		return err
 	}
 	accessor.SetNamespace(namespace)
 	accessor.SetName(name)
-	return c.client.Delete(ctx, obj)
+	return c.client.Delete(ctx, obj, opts...)
 }
 
 func (c *genericClient) List(ctx context.Context, obj runtime.Object, namespace string, opts ...client.ListOption) error {

pkg/controller/sync/controller.go

@@ -39,6 +39,8 @@ import (
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"sigs.k8s.io/kubefed/pkg/apis/core/typeconfig"
 	fedv1b1 "sigs.k8s.io/kubefed/pkg/apis/core/v1beta1"
 	genericclient "sigs.k8s.io/kubefed/pkg/client/generic"
@@ -481,8 +483,16 @@ func (s *KubeFedSyncController) ensureDeletion(fedResource FederatedResource) ut
 		return util.StatusAllOK
 	}
 
+	klog.V(2).Infof("Deserializing delete options of %s %q", kind, key)
+	opts, err := util.GetDeleteOptions(obj)
+	if err != nil {
+		wrappedErr := errors.Wrapf(err, "failed to deserialize delete options of %s %q", kind, key)
+		runtime.HandleError(wrappedErr)
+		return util.StatusError
+	}
+
 	klog.V(2).Infof("Deleting resources managed by %s %q from member clusters.", kind, key)
-	recheckRequired, err := s.deleteFromClusters(fedResource)
+	recheckRequired, err := s.deleteFromClusters(fedResource, opts...)
 	if err != nil {
 		wrappedErr := errors.Wrapf(err, "failed to delete %s %q", kind, key)
 		runtime.HandleError(wrappedErr)
@@ -513,7 +523,7 @@ func (s *KubeFedSyncController) removeManagedLabel(gvk schema.GroupVersionKind,
 	return nil
 }
 
-func (s *KubeFedSyncController) deleteFromClusters(fedResource FederatedResource) (bool, error) {
+func (s *KubeFedSyncController) deleteFromClusters(fedResource FederatedResource, opts ...client.DeleteOption) (bool, error) {
 	gvk := fedResource.TargetGVK()
 	qualifiedName := fedResource.TargetName()
 
@@ -542,7 +552,7 @@ func (s *KubeFedSyncController) deleteFromClusters(fedResource FederatedResource
 			// namespace is no longer cached.
 			dispatcher.RemoveManagedLabel(clusterName, clusterObj)
 		} else {
-			dispatcher.Delete(clusterName)
+			dispatcher.Delete(clusterName, opts...)
 		}
 	})
 	if err != nil {

pkg/controller/sync/dispatch/managed.go

@@ -32,6 +32,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"sigs.k8s.io/kubefed/pkg/client/generic"
 	"sigs.k8s.io/kubefed/pkg/controller/sync/status"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
@@ -245,10 +247,10 @@ func (d *managedDispatcherImpl) Update(clusterName string, clusterObj *unstructu
 	})
 }
 
-func (d *managedDispatcherImpl) Delete(clusterName string) {
+func (d *managedDispatcherImpl) Delete(clusterName string, opts ...client.DeleteOption) {
 	d.RecordStatus(clusterName, status.DeletionTimedOut, nil)
 
-	d.unmanagedDispatcher.Delete(clusterName)
+	d.unmanagedDispatcher.Delete(clusterName, opts...)
 }
 
 func (d *managedDispatcherImpl) RemoveManagedLabel(clusterName string, clusterObj *unstructured.Unstructured) {

pkg/controller/sync/dispatch/unmanaged.go

@@ -28,6 +28,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/klog"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"sigs.k8s.io/kubefed/pkg/client/generic"
 	"sigs.k8s.io/kubefed/pkg/controller/sync/status"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
@@ -41,7 +43,7 @@ const eventTemplate = "%s %s %q in cluster %q"
 type UnmanagedDispatcher interface {
 	OperationDispatcher
 
-	Delete(clusterName string)
+	Delete(clusterName string, opts ...client.DeleteOption)
 	RemoveManagedLabel(clusterName string, clusterObj *unstructured.Unstructured)
 }
 
@@ -72,7 +74,7 @@ func (d *unmanagedDispatcherImpl) Wait() (bool, error) {
 	return d.dispatcher.Wait()
 }
 
-func (d *unmanagedDispatcherImpl) Delete(clusterName string) {
+func (d *unmanagedDispatcherImpl) Delete(clusterName string, opts ...client.DeleteOption) {
 	start := time.Now()
 	d.dispatcher.incrementOperationsInitiated()
 	const op = "delete"
@@ -87,7 +89,7 @@ func (d *unmanagedDispatcherImpl) Delete(clusterName string) {
 
 		obj := &unstructured.Unstructured{}
 		obj.SetGroupVersionKind(d.targetGVK)
-		err := client.Delete(context.Background(), obj, targetName.Namespace, targetName.Name)
+		err := client.Delete(context.Background(), obj, targetName.Namespace, targetName.Name, opts...)
 		if apierrors.IsNotFound(err) {
 			err = nil
 		}

pkg/controller/util/deletionannotation.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// DeleteOptionAnnotation contains options for delete
+	// while deleting resources for member clusters.
+	DeleteOptionAnnotation = "kubefed.io/deleteoption"
+)
+
+// GetDeleteOptions return delete options from the annotation
+func GetDeleteOptions(obj *unstructured.Unstructured) ([]client.DeleteOption, error) {
+	options := make([]client.DeleteOption, 0)
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		return options, nil
+	}
+
+	if optStr, ok := annotations[DeleteOptionAnnotation]; ok {
+		opt := &metav1.DeleteOptions{}
+		if err := json.Unmarshal([]byte(optStr), opt); err != nil {
+			return nil, errors.Wrapf(err, "could not deserialize delete options from annotation value '%s'", optStr)
+		}
+		clientOpt := &client.DeleteOptions{}
+		clientOpt.GracePeriodSeconds = opt.GracePeriodSeconds
+		clientOpt.PropagationPolicy = opt.PropagationPolicy
+		clientOpt.Preconditions = opt.Preconditions
+		options = append(options, clientOpt)
+	}
+	return options, nil
+}
+
+// ApplyDeleteOptions set the DeleteOptions on the annotation
+func ApplyDeleteOptions(obj *unstructured.Unstructured, opts ...client.DeleteOption) error {
+	opt := client.DeleteOptions{}
+	opt.ApplyOptions(opts)
+	deleteOpts := opt.AsDeleteOptions()
+	optBytes, err := json.Marshal(deleteOpts)
+	if err != nil {
+		return errors.Wrapf(err, "could not serialize delete options from object '%v'", deleteOpts)
+	}
+
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = make(map[string]string)
+	}
+	annotations[DeleteOptionAnnotation] = string(optBytes)
+	obj.SetAnnotations(annotations)
+	return nil
+}

pkg/controller/util/deletionannotation_test.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func TestDeleteOptions(t *testing.T) {
+	fedObj := &unstructured.Unstructured{}
+
+	fedObjOrphan := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"metadata": map[string]interface{}{
+				"annotations": map[string]interface{}{
+					DeleteOptionAnnotation: "{\"propagationPolicy\":\"Orphan\"}",
+				},
+			},
+		},
+	}
+
+	fedObjGrace := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"metadata": map[string]interface{}{
+				"annotations": map[string]interface{}{
+					DeleteOptionAnnotation: "{\"gracePeriodSeconds\":5}",
+				},
+			},
+		},
+	}
+
+	fedObjGraceAndOrphan := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"metadata": map[string]interface{}{
+				"annotations": map[string]interface{}{
+					DeleteOptionAnnotation: "{\"propagationPolicy\":\"Orphan\",\"gracePeriodSeconds\":5}",
+				},
+			},
+		},
+	}
+
+	actOpt0 := client.DeleteOptions{}
+	opt0, _ := GetDeleteOptions(fedObj)
+	actOpt0.ApplyOptions(opt0)
+	expOpt0 := client.DeleteOptions{}
+	assert.Equal(t, expOpt0.AsDeleteOptions(), actOpt0.AsDeleteOptions())
+
+	actOpt1 := client.DeleteOptions{}
+	opt1, _ := GetDeleteOptions(fedObjOrphan)
+	actOpt1.ApplyOptions(opt1)
+	prop := metav1.DeletePropagationOrphan
+	expOpt1 := client.DeleteOptions{PropagationPolicy: &prop}
+	assert.Equal(t, expOpt1.AsDeleteOptions(), actOpt1.AsDeleteOptions())
+
+	actOpt2 := client.DeleteOptions{}
+	opt2, _ := GetDeleteOptions(fedObjGrace)
+	actOpt2.ApplyOptions(opt2)
+	seconds := int64(5)
+	expOpt2 := client.DeleteOptions{GracePeriodSeconds: &seconds}
+	assert.Equal(t, expOpt2.AsDeleteOptions(), actOpt2.AsDeleteOptions())
+
+	actOpt3 := client.DeleteOptions{}
+	opt3, _ := GetDeleteOptions(fedObjGraceAndOrphan)
+	actOpt3.ApplyOptions(opt3)
+	expOpt3 := client.DeleteOptions{GracePeriodSeconds: &seconds, PropagationPolicy: &prop}
+	assert.Equal(t, expOpt3.AsDeleteOptions(), actOpt3.AsDeleteOptions())
+}

test/common/crudtester.go

@@ -25,6 +25,7 @@ import (
 	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/pkg/errors"
 
+	appsv1 "k8s.io/api/apps/v1"
 	apiv1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +36,8 @@ import (
 	kubeclientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"sigs.k8s.io/kubefed/pkg/apis/core/common"
 	"sigs.k8s.io/kubefed/pkg/apis/core/typeconfig"
 	fedv1a1 "sigs.k8s.io/kubefed/pkg/apis/core/v1alpha1"
@@ -362,6 +365,66 @@ func (c *FederatedTypeCrudTester) CheckDelete(fedObject *unstructured.Unstructur
 	}
 }
 
+func (c *FederatedTypeCrudTester) SetDeleteOption(fedObject *unstructured.Unstructured, opts ...client.DeleteOption) {
+	apiResource := c.typeConfig.GetFederatedType()
+	qualifiedName := util.NewQualifiedName(fedObject)
+	kind := apiResource.Kind
+	_, err := c.updateObject(apiResource, fedObject, func(obj *unstructured.Unstructured) {
+		err := util.ApplyDeleteOptions(obj, opts...)
+		if err != nil {
+			c.tl.Fatalf("Error apply delete options for %s %q: %v", kind, qualifiedName, err)
+		}
+	})
+	if err != nil {
+		c.tl.Fatalf("Error updating %s %q: %v", kind, qualifiedName, err)
+	}
+}
+
+func (c *FederatedTypeCrudTester) CheckReplicaSet(fedObject *unstructured.Unstructured) {
+	lb, ok, _ := unstructured.NestedStringMap(fedObject.Object, "spec", "selector", "matchLabels")
+	if !ok {
+		c.tl.Fatal("Failed to get matchLabels on the target deployment")
+	}
+
+	matchingLabels := (client.MatchingLabels)(lb)
+
+	for clusterName := range c.testClusters {
+		clusterConfig := c.testClusters[clusterName].Config
+
+		kubeClient := kubeclientset.NewForConfigOrDie(clusterConfig)
+		WaitForNamespaceOrDie(c.tl, kubeClient, clusterName, fedObject.GetNamespace(),
+			c.waitInterval, 30*time.Second)
+
+		clusterClient := genericclient.NewForConfigOrDie(clusterConfig)
+
+		c.tl.Log("Checking that the ReplicaSet still exists in every cluster")
+
+		err := wait.PollImmediate(c.waitInterval, wait.ForeverTestTimeout, func() (bool, error) {
+			objList := &appsv1.ReplicaSetList{}
+			err := clusterClient.List(context.TODO(), objList, fedObject.GetNamespace(), matchingLabels)
+			if err != nil {
+				return false, errors.Errorf("Error retrieving ReplicatSet: %v", err)
+			}
+
+			if len(objList.Items) == 0 {
+				return false, errors.Errorf("ReplicatSet was unexpectedly deleted from cluster %q", clusterName)
+			}
+
+			c.tl.Log("Checking that OwnerReferences has been removed from the ReplicaSet")
+			hasOwner := false
+			for _, rs := range objList.Items {
+				if len(rs.OwnerReferences) > 0 {
+					hasOwner = true
+				}
+			}
+			return !hasOwner, nil
+		})
+		if err != nil {
+			c.tl.Fatalf("Failed to confirm whether ReplicatSet is in cluster %q: %v", clusterName, err)
+		}
+	}
+}
+
 // CheckPropagation checks propagation for the crud tester's clients
 func (c *FederatedTypeCrudTester) CheckPropagation(fedObject *unstructured.Unstructured) {
 	federatedKind := c.typeConfig.GetFederatedType().Kind