Make ConfigReconciler watches for custom resources

This PR makes ConfigReconciler watches for custom resources and triggers reconciliation of the config singleton when any custom resource is created/updated.

Tested: unit tests, GKE cluster

part of kubernetes-retired#488

incubator/hnc/cmd/manager/main.go

@@ -24,6 +24,7 @@ import (
 	prom "github.com/prometheus/client_golang/prometheus"
 	"go.opencensus.io/stats/view"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -50,6 +51,7 @@ func init() {
 
 	_ = api.AddToScheme(scheme)
 	_ = corev1.AddToScheme(scheme)
+	_ = v1beta1.AddToScheme(scheme)
 	// +kubebuilder:scaffold:scheme
 }
 

incubator/hnc/go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/spf13/cobra v0.0.5
 	go.opencensus.io v0.22.3
 	k8s.io/api v0.17.3
+	k8s.io/apiextensions-apiserver v0.17.3
 	k8s.io/apimachinery v0.17.3
 	k8s.io/cli-runtime v0.17.3
 	k8s.io/client-go v0.17.3

incubator/hnc/go.sum

@@ -797,6 +797,8 @@ k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:Ixke
 k8s.io/apiextensions-apiserver v0.17.0/go.mod h1:XiIFUakZywkUl54fVXa7QTEHcqQz9HG55nHd1DCoHj8=
 k8s.io/apiextensions-apiserver v0.17.2 h1:cP579D2hSZNuO/rZj9XFRzwJNYb41DbNANJb6Kolpss=
 k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs=
+k8s.io/apiextensions-apiserver v0.17.3 h1:WDZWkPcbgvchEdDd7ysL21GGPx3UKZQLDZXEkevT6n4=
+k8s.io/apiextensions-apiserver v0.17.3/go.mod h1:CJbCyMfkKftAd/X/V6OTHYhVn7zXnDdnkUjS1h0GTeY=
 k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
 k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d h1:Jmdtdt1ZnoGfWWIIik61Z7nKYgO3J+swQJtPYsP9wHA=
 k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
@@ -809,6 +811,7 @@ k8s.io/apimachinery v0.17.3 h1:f+uZV6rm4/tHE7xXgLyToprg6xWairaClGVkm2t8omg=
 k8s.io/apimachinery v0.17.3/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g=
 k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg=
 k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo=
+k8s.io/apiserver v0.17.3/go.mod h1:iJtsPpu1ZpEnHaNawpSV0nYTGBhhX2dUlnn7/QS7QiY=
 k8s.io/cli-runtime v0.0.0-20190314001948-2899ed30580f h1:gRAqn9Z3rp62UwLU3PdC7Lhmsvd3e9PXLsq7EG+bq1s=
 k8s.io/cli-runtime v0.0.0-20190314001948-2899ed30580f/go.mod h1:qWnH3/b8sp/l7EvlDh7ulDU3UWA4P4N1NFbEEP791tM=
 k8s.io/cli-runtime v0.0.0-20191214191754-e6dc6d5c8724/go.mod h1:wzlq80lvjgHW9if6MlE4OIGC86MDKsy5jtl9nxz/IYY=
@@ -829,10 +832,12 @@ k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible/go.mod h1:7v
 k8s.io/code-generator v0.0.0-20191214185510-0b9b3c99f9f2/go.mod h1:BjGKcoq1MRUmcssvHiSxodCco1T6nVIt4YeCT5CMSao=
 k8s.io/code-generator v0.17.0/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s=
 k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s=
+k8s.io/code-generator v0.17.3/go.mod h1:l8BLVwASXQZTo2xamW5mQNFCe1XPiAesVq7Y1t7PiQQ=
 k8s.io/component-base v0.0.0-20191214190519-d868452632e2/go.mod h1:wupxkh1T/oUDqyTtcIjiEfpbmIHGm8By/vqpSKC6z8c=
 k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc=
 k8s.io/component-base v0.17.2 h1:0XHf+cerTvL9I5Xwn9v+0jmqzGAZI7zNydv4tL6Cw6A=
 k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs=
+k8s.io/component-base v0.17.3/go.mod h1:GeQf4BrgelWm64PXkIXiPh/XS0hnO42d9gx9BtbZRp8=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=

incubator/hnc/pkg/reconcilers/hnc_config.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"fmt"
 
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	"github.com/go-logr/logr"
@@ -293,10 +295,21 @@ func (r *ConfigReconciler) forceInitialReconcile(log logr.Logger, reason string)
 
 // SetupWithManager builds a controller with the reconciler.
 func (r *ConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	// Whenever a CRD is created/updated, we will send a request to reconcile the
+	// singleton again, in case the singleton has configuration for the resource.
+	crdMapFn := handler.ToRequestsFunc(
+		func(a handler.MapObject) []reconcile.Request {
+			return []reconcile.Request{
+				{NamespacedName: types.NamespacedName{
+					Name: api.HNCConfigSingleton,
+				}},
+			}
+		})
 	err := ctrl.NewControllerManagedBy(mgr).
 		For(&api.HNCConfiguration{}).
 		Watches(&source.Channel{Source: r.Igniter}, &handler.EnqueueRequestForObject{}).
-		Complete(r)
+		Watches(&source.Kind{Type: &v1beta1.CustomResourceDefinition{}},
+			&handler.EnqueueRequestsFromMapFunc{ToRequests: crdMapFn}).Complete(r)
 	if err != nil {
 		return err
 	}

incubator/hnc/pkg/reconcilers/hnc_config_test.go

@@ -9,6 +9,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	v1 "k8s.io/api/rbac/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -212,6 +213,25 @@ var _ = Describe("HNCConfiguration", func() {
 		Expect(hasObject(ctx, "Secret", barName, "foo-sec-2")()).Should(BeFalse())
 
 	})
+
+	It("should reconcile after adding a new crd to the apiserver", func() {
+		// Add a config for a type that hasn't been defined yet.
+		addToHNCConfig(ctx, "stable.example.com/v1", "CronTab", api.Propagate)
+
+		// The corresponding object reconciler should not be created because the type does not exist.
+		Eventually(hasHNCConfigurationConditionWithMsg(ctx, api.ObjectReconcilerCreationFailed, "stable.example.com/v1, Kind=CronTab")).Should(BeTrue())
+
+		// Add the CRD for CronTab to the apiserver.
+		createCronTabCRD(ctx)
+		setParent(ctx, barName, fooName)
+		makeObject(ctx, "CronTab", fooName, "foo-crontab")
+
+		// The object reconciler for CronTab should be created successfully.
+		Eventually(hasHNCConfigurationConditionWithMsg(ctx, api.ObjectReconcilerCreationFailed, "stable.example.com/v1, Kind=CronTab")).Should(BeFalse())
+		// "foo-crontab" should be propagated from foo to bar.
+		Eventually(hasObject(ctx, "CronTab", barName, "foo-crontab")).Should(BeTrue())
+		Expect(objectInheritedFrom(ctx, "CronTab", barName, "foo-crontab")).Should(Equal(fooName))
+	})
 })
 
 func hasTypeWithMode(apiVersion, kind string, mode api.SynchronizationMode, config *api.HNCConfiguration) func() bool {
@@ -312,3 +332,18 @@ func removeHNCConfigType(ctx context.Context, apiVersion, kind string) {
 		return updateHNCConfig(ctx, c)
 	}).Should(Succeed())
 }
+
+func createCronTabCRD(ctx context.Context) {
+	crontab := v1beta1.CustomResourceDefinition{}
+	crontab.Kind = "CronTab"
+	crontab.APIVersion = "stable.example.com/v1"
+	crontab.Name = "crontabs.stable.example.com"
+	crontab.Spec.Group = "stable.example.com"
+	crontab.Spec.Versions = append(crontab.Spec.Versions, v1beta1.CustomResourceDefinitionVersion{Name: "v1", Served: true, Storage: true})
+	crontab.Spec.Names.Kind = "CronTab"
+	crontab.Spec.Names.Plural = "crontabs"
+	crontab.Spec.Names.Singular = "crontab"
+	Eventually(func() error {
+		return k8sClient.Create(ctx, &crontab)
+	}).Should(Succeed())
+}

incubator/hnc/pkg/reconcilers/suite_test.go

@@ -25,6 +25,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -88,6 +89,9 @@ var _ = BeforeSuite(func(done Done) {
 	err = corev1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = v1beta1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
 	// +kubebuilder:scaffold:scheme
 
 	// CF: https://github.com/microsoft/azure-databricks-operator/blob/0f722a710fea06b86ecdccd9455336ca712bf775/controllers/suite_test.go

incubator/hnc/pkg/reconcilers/test_helpers_test.go

@@ -24,6 +24,7 @@ var GVKs = map[string]schema.GroupVersionKind{
 	"ResourceQuota": {Group: "", Version: "v1", Kind: "ResourceQuota"},
 	"LimitRange":    {Group: "", Version: "v1", Kind: "LimitRange"},
 	"ConfigMap":     {Group: "", Version: "v1", Kind: "ConfigMap"},
+	"CronTab":       {Group: "stable.example.com", Version: "v1", Kind: "CronTab"},
 }
 
 func setParent(ctx context.Context, nm string, pnm string) {