Merge pull request #429 from weaveworks/alerts

Implement canary alerts and alert providers
fluxcd · Feb 11, 2020 · ad68ca3 · ad68ca3
2 parents 963a9af + fd85a34
commit ad68ca3
Show file tree

Hide file tree

Showing 46 changed files with 1,960 additions and 388 deletions.
diff --git a/artifacts/flagger/account.yaml b/artifacts/flagger/account.yaml
@@ -44,6 +44,8 @@ rules:
       - canaries/status
       - metrictemplates
       - metrictemplates/status
+      - alertproviders
+      - alertproviders/status
     verbs: ["*"]
   - apiGroups:
       - networking.istio.io

diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml
@@ -591,3 +591,62 @@ spec:
             query:
               description: Query of this metric template
               type: string
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: alertproviders.flagger.app
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  group: flagger.app
+  version: v1beta1
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+  names:
+    plural: alertproviders
+    singular: alertprovider
+    kind: AlertProvider
+    categories:
+      - all
+  scope: Namespaced
+  subresources:
+    status: {}
+  additionalPrinterColumns:
+    - name: Type
+      type: string
+      JSONPath: .spec.type
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          oneOf:
+            - required:
+              - type
+              - address
+            - required:
+              - type
+              - secretRef
+          properties:
+            type:
+              description: Type of this provider
+              type: string
+              enum:
+                - slack
+                - msteams
+                - discord
+                - rocket
+            address:
+              description: Hook URL address of this provider
+              type: string
+            secretRef:
+              description: Kubernetes secret reference containing the provider address
+              type: object
+              required:
+                - name
+              properties:
+                name:
+                  description: Name of the Kubernetes secret
+                  type: string
diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml
@@ -592,4 +592,63 @@ spec:
             query:
               description: Query of this metric template
               type: string
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: alertproviders.flagger.app
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  group: flagger.app
+  version: v1beta1
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+  names:
+    plural: alertproviders
+    singular: alertprovider
+    kind: AlertProvider
+    categories:
+      - all
+  scope: Namespaced
+  subresources:
+    status: {}
+  additionalPrinterColumns:
+    - name: Type
+      type: string
+      JSONPath: .spec.type
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          oneOf:
+            - required:
+              - type
+              - address
+            - required:
+              - type
+              - secretRef
+          properties:
+            type:
+              description: Type of this provider
+              type: string
+              enum:
+                - slack
+                - msteams
+                - discord
+                - rocket
+            address:
+              description: Hook URL address of this provider
+              type: string
+            secretRef:
+              description: Kubernetes secret reference containing the provider address
+              type: object
+              required:
+                - name
+              properties:
+                name:
+                  description: Name of the Kubernetes secret
+                  type: string
 {{- end }}
diff --git a/charts/flagger/templates/rbac.yaml b/charts/flagger/templates/rbac.yaml
@@ -40,6 +40,8 @@ rules:
       - canaries/status
       - metrictemplates
       - metrictemplates/status
+      - alertproviders
+      - alertproviders/status
     verbs: ["*"]
   - apiGroups:
       - networking.istio.io

diff --git a/cmd/flagger/main.go b/cmd/flagger/main.go
@@ -129,6 +129,7 @@ func main() {
 
 	verifyCRDs(flaggerClient, logger)
 	verifyKubernetesVersion(kubeClient, logger)
+	infos := startInformers(flaggerClient, logger, stopCh)
 
 	labels := strings.Split(selectorLabels, ",")
 	if len(labels) < 1 {
@@ -157,23 +158,6 @@ func main() {
 	// start HTTP server
 	go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
 
-	// start informers
-	flaggerInformerFactory := informers.NewSharedInformerFactoryWithOptions(flaggerClient, time.Second*30, informers.WithNamespace(namespace))
-
-	logger.Info("Waiting for canary informer cache to sync")
-	canaryInformer := flaggerInformerFactory.Flagger().V1beta1().Canaries()
-	go canaryInformer.Informer().Run(stopCh)
-	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, canaryInformer.Informer().HasSynced); !ok {
-		logger.Fatalf("failed to wait for cache to sync")
-	}
-
-	logger.Info("Waiting for metric template informer cache to sync")
-	metricInformer := flaggerInformerFactory.Flagger().V1beta1().MetricTemplates()
-	go metricInformer.Informer().Run(stopCh)
-	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, metricInformer.Informer().HasSynced); !ok {
-		logger.Fatalf("failed to wait for cache to sync")
-	}
-
 	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, logger, meshClient)
 
 	var configTracker canary.Tracker
@@ -193,7 +177,7 @@ func main() {
 		kubeClient,
 		meshClient,
 		flaggerClient,
-		canaryInformer,
+		infos,
 		controlLoopInterval,
 		logger,
 		notifierClient,
@@ -237,6 +221,37 @@ func main() {
 	}
 }
 
+func startInformers(flaggerClient clientset.Interface, logger *zap.SugaredLogger, stopCh <-chan struct{}) controller.Informers {
+	flaggerInformerFactory := informers.NewSharedInformerFactoryWithOptions(flaggerClient, time.Second*30, informers.WithNamespace(namespace))
+
+	logger.Info("Waiting for canary informer cache to sync")
+	canaryInformer := flaggerInformerFactory.Flagger().V1beta1().Canaries()
+	go canaryInformer.Informer().Run(stopCh)
+	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, canaryInformer.Informer().HasSynced); !ok {
+		logger.Fatalf("failed to wait for cache to sync")
+	}
+
+	logger.Info("Waiting for metric template informer cache to sync")
+	metricInformer := flaggerInformerFactory.Flagger().V1beta1().MetricTemplates()
+	go metricInformer.Informer().Run(stopCh)
+	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, metricInformer.Informer().HasSynced); !ok {
+		logger.Fatalf("failed to wait for cache to sync")
+	}
+
+	logger.Info("Waiting for alert provider informer cache to sync")
+	alertInformer := flaggerInformerFactory.Flagger().V1beta1().AlertProviders()
+	go alertInformer.Informer().Run(stopCh)
+	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, alertInformer.Informer().HasSynced); !ok {
+		logger.Fatalf("failed to wait for cache to sync")
+	}
+
+	return controller.Informers{
+		CanaryInformer: canaryInformer,
+		MetricInformer: metricInformer,
+		AlertInformer:  alertInformer,
+	}
+}
+
 func startLeaderElection(ctx context.Context, run func(), ns string, kubeClient kubernetes.Interface, logger *zap.SugaredLogger) {
 	configMapName := "flagger-leader-election"
 	id, err := os.Hostname()
@@ -322,6 +337,11 @@ func verifyCRDs(flaggerClient clientset.Interface, logger *zap.SugaredLogger) {
 	if err != nil {
 		logger.Fatalf("MetricTemplate CRD is not registered %v", err)
 	}
+
+	_, err = flaggerClient.FlaggerV1beta1().AlertProviders(namespace).List(metav1.ListOptions{Limit: 1})
+	if err != nil {
+		logger.Fatalf("AlertProvider CRD is not registered %v", err)
+	}
 }
 
 func verifyKubernetesVersion(kubeClient kubernetes.Interface, logger *zap.SugaredLogger) {

diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml
@@ -591,3 +591,62 @@ spec:
             query:
               description: Query of this metric template
               type: string
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: alertproviders.flagger.app
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  group: flagger.app
+  version: v1beta1
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+  names:
+    plural: alertproviders
+    singular: alertprovider
+    kind: AlertProvider
+    categories:
+      - all
+  scope: Namespaced
+  subresources:
+    status: {}
+  additionalPrinterColumns:
+    - name: Type
+      type: string
+      JSONPath: .spec.type
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          oneOf:
+            - required:
+              - type
+              - address
+            - required:
+              - type
+              - secretRef
+          properties:
+            type:
+              description: Type of this provider
+              type: string
+              enum:
+                - slack
+                - msteams
+                - discord
+                - rocket
+            address:
+              description: Hook URL address of this provider
+              type: string
+            secretRef:
+              description: Kubernetes secret reference containing the provider address
+              type: object
+              required:
+                - name
+              properties:
+                name:
+                  description: Name of the Kubernetes secret
+                  type: string
diff --git a/kustomize/base/flagger/rbac.yaml b/kustomize/base/flagger/rbac.yaml
@@ -34,6 +34,8 @@ rules:
       - canaries/status
       - metrictemplates
       - metrictemplates/status
+      - alertproviders
+      - alertproviders/status
     verbs: ["*"]
   - apiGroups:
       - networking.istio.io