Invalidate discovery cache periodically

The cluster resource definitions may not be stable throughout the life of the
cluster.

We were already invalidating the cache every time a CRD object was altered and
replaying the CRD changes every 5 minutes.

However, that's not enough because CRD-unrelated changes seem to happen when the
cluster bootstraps.

cluster/kubernetes/cached_disco.go

@@ -96,7 +96,32 @@ func makeCachedDiscovery(d discovery.DiscoveryInterface, c crd.Interface, shutdo
 	}
 
 	handler := handlerFn(cachedDisco)
-	store, controller := toolscache.NewInformer(lw, &crdv1beta1.CustomResourceDefinition{}, 5*time.Minute, handler)
+	store, controller := toolscache.NewInformer(lw, &crdv1beta1.CustomResourceDefinition{}, 0, handler)
+	go cachedDisco.invalidatePeriodically(shutdown)
 	go controller.Run(shutdown)
 	return cachedDisco, store, controller
 }
+
+func (d *cachedDiscovery) invalidatePeriodically(shutdown <-chan struct{}) {
+	// Make the first period shorter since we may be bootstrapping a
+	// newly-created cluster and the resource definitions may not be
+	// complete/stable yet.
+	initialPeriodDuration := 1 * time.Minute
+	subsequentPeriodsDuration := 5 * initialPeriodDuration
+	isInitialPeriod := true
+	ticker := time.NewTicker(initialPeriodDuration)
+	for {
+		select {
+		case <-ticker.C:
+			if isInitialPeriod {
+				ticker.Stop()
+				ticker = time.NewTicker(subsequentPeriodsDuration)
+			}
+			isInitialPeriod = false
+			d.Invalidate()
+		case <-shutdown:
+			ticker.Stop()
+			return
+		}
+	}
+}