use TLS certificates for metrics scraping from Thanos Querier

openshift · Sep 30, 2021 · 191860e · 191860e
1 parent 8b66167
commit 191860e
Show file tree

Hide file tree

Showing 7 changed files with 163 additions and 31 deletions.
diff --git a/assets/thanos-querier/deployment.yaml b/assets/thanos-querier/deployment.yaml
@@ -191,6 +191,30 @@ spec:
           name: secret-thanos-querier-tls
         - mountPath: /etc/kube-rbac-proxy
           name: secret-thanos-querier-kube-rbac-proxy-rules
+      - args:
+        - --secure-listen-address=0.0.0.0:9097
+        - --upstream=http://127.0.0.1:9090
+        - --config-file=/etc/kube-rbac-proxy/config.yaml
+        - --tls-cert-file=/etc/tls/private/tls.crt
+        - --tls-private-key-file=/etc/tls/private/tls.key
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+        - --logtostderr=true
+        - --allow-paths=/metrics
+        image: quay.io/brancz/kube-rbac-proxy:v0.11.0
+        name: kube-rbac-proxy-metrics
+        ports:
+        - containerPort: 9097
+          name: metrics
+        resources:
+          requests:
+            cpu: 1m
+            memory: 15Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /etc/tls/private
+          name: secret-thanos-querier-tls
+        - mountPath: /etc/kube-rbac-proxy
+          name: secret-thanos-querier-kube-rbac-proxy-metrics
       nodeSelector:
         kubernetes.io/os: linux
       priorityClassName: system-cluster-critical
@@ -209,3 +233,6 @@ spec:
       - name: secret-thanos-querier-kube-rbac-proxy-rules
         secret:
           secretName: thanos-querier-kube-rbac-proxy-rules
+      - name: secret-thanos-querier-kube-rbac-proxy-metrics
+        secret:
+          secretName: thanos-querier-kube-rbac-proxy-metrics
diff --git a/assets/thanos-querier/kube-rbac-proxy-metric-secret.yaml b/assets/thanos-querier/kube-rbac-proxy-metric-secret.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+data: {}
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/name: thanos-query
+  name: thanos-querier-kube-rbac-proxy-metrics
+  namespace: openshift-monitoring
+stringData:
+  config.yaml: |-
+    "authorization":
+      "static":
+      - "path": "/metrics"
+        "resourceRequest": false
+        "user":
+          "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s"
+        "verb": "get"
+type: Opaque
diff --git a/assets/thanos-querier/service-monitor.yaml b/assets/thanos-querier/service-monitor.yaml
@@ -13,7 +13,7 @@ spec:
   endpoints:
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     interval: 30s
-    port: web
+    port: metrics
     scheme: https
     tlsConfig:
       caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt

diff --git a/assets/thanos-querier/service.yaml b/assets/thanos-querier/service.yaml
@@ -22,6 +22,9 @@ spec:
   - name: tenancy-rules
     port: 9093
     targetPort: tenancy-rules
+  - name: metrics
+    port: 9097
+    targetPort: metrics
   selector:
     app.kubernetes.io/component: query-layer
     app.kubernetes.io/instance: thanos-querier

diff --git a/jsonnet/components/thanos-querier.libsonnet b/jsonnet/components/thanos-querier.libsonnet
@@ -1,3 +1,4 @@
+local generateSecret = import '../utils/generate-secret.libsonnet';
 local querier = import 'github.com/thanos-io/kube-thanos/jsonnet/kube-thanos/kube-thanos-query.libsonnet';
 
 function(params)
@@ -179,6 +180,12 @@ function(params)
       },
     },
 
+    kubeRbacProxyMetricSecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'thanos-querier-kube-rbac-proxy-metrics') + {
+      metadata+: {
+        labels: { 'app.kubernetes.io/name': 'thanos-query' },
+      },
+    },
+
     // Same as kubeRbacProxySecret but performs a SubjectAccessReview
     // asserting if the request bearer token in flight has permissions
     // to access the prometheusrules.monitoring.coreos.com API.
@@ -233,19 +240,28 @@ function(params)
         labels: tq.config.commonLabels,
       },
       spec+: {
-        ports: [{
-          name: 'web',
-          port: 9091,
-          targetPort: 'web',
-        }, {
-          name: 'tenancy',
-          port: 9092,
-          targetPort: 'tenancy',
-        }, {
-          name: 'tenancy-rules',
-          port: 9093,
-          targetPort: 'tenancy-rules',
-        }],
+        ports: [
+          {
+            name: 'web',
+            port: 9091,
+            targetPort: 'web',
+          },
+          {
+            name: 'tenancy',
+            port: 9092,
+            targetPort: 'tenancy',
+          },
+          {
+            name: 'tenancy-rules',
+            port: 9093,
+            targetPort: 'tenancy-rules',
+          },
+          {
+            name: 'metrics',
+            port: 9097,
+            targetPort: 'metrics',
+          },
+        ],
         type: 'ClusterIP',
       },
     },
@@ -255,7 +271,7 @@ function(params)
         spec+: {
           endpoints: [
             {
-              port: 'web',
+              port: 'metrics',
               interval: '30s',
               scheme: 'https',
               tlsConfig: {
@@ -342,6 +358,12 @@ function(params)
                   secretName: 'thanos-querier-kube-rbac-proxy-rules',
                 },
               },
+              {
+                name: 'secret-thanos-querier-kube-rbac-proxy-metrics',
+                secret: {
+                  secretName: 'thanos-querier-kube-rbac-proxy-metrics',
+                },
+              },
             ],
             serviceAccountName: 'thanos-querier',
             priorityClassName: 'system-cluster-critical',
@@ -551,6 +573,43 @@ function(params)
                   },
                 ],
               },
+              {
+                name: 'kube-rbac-proxy-metrics',
+                image: cfg.kubeRbacProxyImage,
+                resources: {
+                  requests: {
+                    memory: '15Mi',
+                    cpu: '1m',
+                  },
+                },
+                ports: [
+                  {
+                    containerPort: 9097,
+                    name: 'metrics',
+                  },
+                ],
+                args: [
+                  '--secure-listen-address=0.0.0.0:9097',
+                  '--upstream=http://127.0.0.1:9090',
+                  '--config-file=/etc/kube-rbac-proxy/config.yaml',
+                  '--tls-cert-file=/etc/tls/private/tls.crt',
+                  '--tls-private-key-file=/etc/tls/private/tls.key',
+                  '--tls-cipher-suites=' + cfg.tlsCipherSuites,
+                  '--logtostderr=true',
+                  '--allow-paths=/metrics',
+                ],
+                terminationMessagePolicy: 'FallbackToLogsOnError',
+                volumeMounts: [
+                  {
+                    mountPath: '/etc/tls/private',
+                    name: 'secret-thanos-querier-tls',
+                  },
+                  {
+                    mountPath: '/etc/kube-rbac-proxy',
+                    name: 'secret-' + $.kubeRbacProxyMetricSecret.metadata.name,
+                  },
+                ],
+              },
             ],
           },
         },

diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go
@@ -204,21 +204,22 @@ var (
 	TelemeterClientServiceMonitor         = "telemeter-client/service-monitor.yaml"
 	TelemeterClientServingCertsCABundle   = "telemeter-client/serving-certs-ca-bundle.yaml"
 
-	ThanosQuerierDeployment           = "thanos-querier/deployment.yaml"
-	ThanosQuerierPodDisruptionBudget  = "thanos-querier/pod-disruption-budget.yaml"
-	ThanosQuerierService              = "thanos-querier/service.yaml"
-	ThanosQuerierServiceMonitor       = "thanos-querier/service-monitor.yaml"
-	ThanosQuerierPrometheusRule       = "thanos-querier/prometheus-rule.yaml"
-	ThanosQuerierRoute                = "thanos-querier/route.yaml"
-	ThanosQuerierOauthCookieSecret    = "thanos-querier/oauth-cookie-secret.yaml"
-	ThanosQuerierHtpasswdSecret       = "thanos-querier/oauth-htpasswd-secret.yaml"
-	ThanosQuerierRBACProxySecret      = "thanos-querier/kube-rbac-proxy-secret.yaml"
-	ThanosQuerierRBACProxyRulesSecret = "thanos-querier/kube-rbac-proxy-rules-secret.yaml"
-	ThanosQuerierServiceAccount       = "thanos-querier/service-account.yaml"
-	ThanosQuerierClusterRole          = "thanos-querier/cluster-role.yaml"
-	ThanosQuerierClusterRoleBinding   = "thanos-querier/cluster-role-binding.yaml"
-	ThanosQuerierGrpcTLSSecret        = "thanos-querier/grpc-tls-secret.yaml"
-	ThanosQuerierTrustedCABundle      = "thanos-querier/trusted-ca-bundle.yaml"
+	ThanosQuerierDeployment             = "thanos-querier/deployment.yaml"
+	ThanosQuerierPodDisruptionBudget    = "thanos-querier/pod-disruption-budget.yaml"
+	ThanosQuerierService                = "thanos-querier/service.yaml"
+	ThanosQuerierServiceMonitor         = "thanos-querier/service-monitor.yaml"
+	ThanosQuerierPrometheusRule         = "thanos-querier/prometheus-rule.yaml"
+	ThanosQuerierRoute                  = "thanos-querier/route.yaml"
+	ThanosQuerierOauthCookieSecret      = "thanos-querier/oauth-cookie-secret.yaml"
+	ThanosQuerierHtpasswdSecret         = "thanos-querier/oauth-htpasswd-secret.yaml"
+	ThanosQuerierRBACProxySecret        = "thanos-querier/kube-rbac-proxy-secret.yaml"
+	ThanosQuerierRBACProxyRulesSecret   = "thanos-querier/kube-rbac-proxy-rules-secret.yaml"
+	ThanosQuerierRBACProxyMetricsSecret = "thanos-querier/kube-rbac-proxy-metric-secret.yaml"
+	ThanosQuerierServiceAccount         = "thanos-querier/service-account.yaml"
+	ThanosQuerierClusterRole            = "thanos-querier/cluster-role.yaml"
+	ThanosQuerierClusterRoleBinding     = "thanos-querier/cluster-role-binding.yaml"
+	ThanosQuerierGrpcTLSSecret          = "thanos-querier/grpc-tls-secret.yaml"
+	ThanosQuerierTrustedCABundle        = "thanos-querier/trusted-ca-bundle.yaml"
 
 	ThanosRulerCustomResource               = "thanos-ruler/thanos-ruler.yaml"
 	ThanosRulerService                      = "thanos-ruler/service.yaml"
@@ -1163,6 +1164,17 @@ func (f *Factory) ThanosQuerierRBACProxyRulesSecret() (*v1.Secret, error) {
 	return s, nil
 }
 
+func (f *Factory) ThanosQuerierRBACProxyMetricsSecret() (*v1.Secret, error) {
+	s, err := f.NewSecret(f.assets.MustNewAssetReader(ThanosQuerierRBACProxyMetricsSecret))
+	if err != nil {
+		return nil, err
+	}
+
+	s.Namespace = f.namespace
+
+	return s, nil
+}
+
 func (f *Factory) PrometheusK8sServingCertsCABundle() (*v1.ConfigMap, error) {
 	c, err := f.NewConfigMap(f.assets.MustNewAssetReader(PrometheusK8sServingCertsCABundle))
 	if err != nil {
@@ -3031,6 +3043,9 @@ func (f *Factory) ThanosQuerierDeployment(grpcTLS *v1.Secret, enableUserWorkload
 
 		case "kube-rbac-proxy-rules":
 			d.Spec.Template.Spec.Containers[i].Image = f.config.Images.KubeRbacProxy
+
+		case "kube-rbac-proxy-metrics":
+			d.Spec.Template.Spec.Containers[i].Image = f.config.Images.KubeRbacProxy
 		}
 	}
 
@@ -3085,7 +3100,7 @@ func (f *Factory) ThanosQuerierServiceMonitor() (*monv1.ServiceMonitor, error) {
 	}
 
 	var found bool
-	const endpointPort = "web"
+	const endpointPort = "metrics"
 	for i := range sm.Spec.Endpoints {
 		if sm.Spec.Endpoints[i].Port == endpointPort {
 			found = true

diff --git a/pkg/tasks/thanos_querier.go b/pkg/tasks/thanos_querier.go
@@ -124,6 +124,16 @@ func (t *ThanosQuerierTask) Run(ctx context.Context) error {
 		return errors.Wrap(err, "creating Thanos Querier RBAC proxy rules Secret failed")
 	}
 
+	rs, err = t.factory.ThanosQuerierRBACProxyMetricsSecret()
+	if err != nil {
+		return errors.Wrap(err, "initializing Thanos Querier RBAC proxy metrics Secret failed")
+	}
+
+	err = t.client.CreateIfNotExistSecret(ctx, rs)
+	if err != nil {
+		return errors.Wrap(err, "creating Thanos Querier RBAC proxy metrics Secret failed")
+	}
+
 	sa, err := t.factory.ThanosQuerierServiceAccount()
 	if err != nil {
 		return errors.Wrap(err, "initializing Thanos Querier ServiceAccount failed")