*: use a bearer token file

For security concerns, it's better to pass the bearer token via a Secret rather than sticking it in the Prometheus custom resource. Signed-off-by: Simon Pasquier <[email protected]>
openshift · Aug 8, 2022 · 73734fb · 73734fb
1 parent ad1ed08
commit 73734fb
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 15 deletions.
diff --git a/assets/prometheus-k8s/telemetry-secret.yaml b/assets/prometheus-k8s/telemetry-secret.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+data: {}
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/name: prometheus-k8s
+  name: telemetry-server
+  namespace: openshift-monitoring
+type: Opaque
diff --git a/hack/local-cmo.sh b/hack/local-cmo.sh
@@ -106,6 +106,7 @@ main(){
     -kubeconfig "${KUBECONFIG}" \
     -namespace=openshift-monitoring \
     -configmap=cluster-monitoring-config \
+    -enabled-remote-write \
     -logtostderr=true -v=4 2>&1 | tee operator.log
 }
 

diff --git a/jsonnet/components/prometheus.libsonnet b/jsonnet/components/prometheus.libsonnet
@@ -217,6 +217,19 @@ function(params)
 
     kubeRbacProxySecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'kube-rbac-proxy'),
 
+    // Secret holding the token to authenticate against the Telemetry server when using native remote-write.
+    telemetrySecret: {
+      apiVersion: 'v1',
+      kind: 'Secret',
+      metadata: {
+        name: 'telemetry-server',
+        namespace: cfg.namespace,
+        labels: { 'app.kubernetes.io/name': 'prometheus-k8s' },
+      },
+      type: 'Opaque',
+      data: {},
+    },
+
     // This changes the Prometheuses to be scraped with TLS, authN and
     // authZ, which are not present in kube-prometheus.
 

diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go
@@ -147,6 +147,7 @@ var (
 	PrometheusK8sThanosSidecarServiceMonitor      = "prometheus-k8s/service-monitor-thanos-sidecar.yaml"
 	PrometheusK8sTAlertmanagerRoleBinding         = "prometheus-k8s/alertmanager-role-binding.yaml"
 	PrometheusK8sPodDisruptionBudget              = "prometheus-k8s/pod-disruption-budget.yaml"
+	PrometheusK8sTelemetry                        = "prometheus-k8s/telemetry-secret.yaml"
 
 	PrometheusUserWorkloadServingCertsCABundle        = "prometheus-user-workload/serving-certs-ca-bundle.yaml"
 	PrometheusUserWorkloadServiceAccount              = "prometheus-user-workload/service-account.yaml"
@@ -290,6 +291,8 @@ var (
 	ControlPlanePrometheusRule        = "control-plane/prometheus-rule.yaml"
 	ControlPlaneKubeletServiceMonitor = "control-plane/service-monitor-kubelet.yaml"
 	ControlPlaneEtcdServiceMonitor    = "control-plane/service-monitor-etcd.yaml"
+
+	telemetryTokenSecretKey = "token"
 )
 
 var (
@@ -1614,7 +1617,29 @@ func (f *Factory) PrometheusK8sTrustedCABundle() (*v1.ConfigMap, error) {
 	return cm, nil
 }
 
-func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.ConfigMap) (*monv1.Prometheus, error) {
+func (f *Factory) PrometheusK8sTelemetrySecret() (*v1.Secret, error) {
+	s, err := f.NewSecret(f.assets.MustNewAssetReader(PrometheusK8sTelemetry))
+	if err != nil {
+		return nil, err
+	}
+	compositeToken, err := json.Marshal(map[string]string{
+		"cluster_id":          f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID,
+		"authorization_token": f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.Token,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	b := make([]byte, base64.StdEncoding.EncodedLen(len(compositeToken)))
+	base64.StdEncoding.Encode(b, compositeToken)
+	s.Data = map[string][]byte{
+		telemetryTokenSecretKey: b,
+	}
+
+	return s, nil
+}
+
+func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.ConfigMap, telemetrySecret *v1.Secret) (*monv1.Prometheus, error) {
 	p, err := f.NewPrometheus(f.assets.MustNewAssetReader(PrometheusK8s))
 	if err != nil {
 		return nil, err
@@ -1664,23 +1689,18 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.Config
 		return nil, err
 	}
 
-	telemetryEnabled := f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.IsEnabled()
 	clusterID := f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.ClusterID
-	if telemetryEnabled && f.config.RemoteWrite {
-
+	if telemetrySecret != nil {
 		selectorRelabelConfig, err := promqlgen.LabelSelectorsToRelabelConfig(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.TelemetryMatches)
 		if err != nil {
 			return nil, errors.Wrap(err, "generate label selector relabel config")
 		}
 
-		compositeToken, err := json.Marshal(map[string]string{
-			"cluster_id":          clusterID,
-			"authorization_token": f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.Token,
-		})
+		p.Spec.Secrets = append(p.Spec.Secrets, telemetrySecret.GetName())
 
 		spec := monv1.RemoteWriteSpec{
-			URL:         f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.TelemeterServerURL,
-			BearerToken: base64.StdEncoding.EncodeToString(compositeToken),
+			URL:             f.config.ClusterMonitoringConfiguration.TelemeterClientConfig.TelemeterServerURL,
+			BearerTokenFile: fmt.Sprintf("/etc/prometheus/secrets/%s/%s", telemetrySecret.GetName(), telemetryTokenSecretKey),
 			QueueConfig: &monv1.QueueConfig{
 				// Amount of samples to load from the WAL into the in-memory
 				// buffer before waiting for samples to be sent successfully
@@ -1723,10 +1743,6 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, trustedCABundleCM *v1.Config
 		}
 
 		p.Spec.RemoteWrite = []monv1.RemoteWriteSpec{spec}
-
-	}
-	if !telemetryEnabled {
-		p.Spec.RemoteWrite = nil
 	}
 
 	if len(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.RemoteWrite) > 0 {

diff --git a/pkg/tasks/prometheus.go b/pkg/tasks/prometheus.go
@@ -306,6 +306,24 @@ func (t *PrometheusTask) Run(ctx context.Context) error {
 		}
 	}
 
+	telemetrySecret, err := t.factory.PrometheusK8sTelemetrySecret()
+	if err != nil {
+		return errors.Wrap(err, "initializing Prometheus telemetry secret failed")
+	}
+
+	if t.config.ClusterMonitoringConfiguration.TelemeterClientConfig.IsEnabled() && t.config.RemoteWrite {
+		klog.V(4).Info("updating Prometheus telemetry secret")
+		if err = t.client.CreateOrUpdateSecret(ctx, telemetrySecret); err != nil {
+			return errors.Wrap(err, "reconciling Prometheus telemetry secret failed")
+		}
+	} else {
+		klog.V(4).Info("deleting Prometheus telemetry secret")
+		if err = t.client.DeleteSecret(ctx, telemetrySecret); err != nil {
+			return errors.Wrap(err, "deleting Prometheus telemetry secret failed")
+		}
+		telemetrySecret = nil
+	}
+
 	{
 		// Create trusted CA bundle ConfigMap.
 		trustedCA, err := t.factory.PrometheusK8sTrustedCABundle()
@@ -334,7 +352,7 @@ func (t *PrometheusTask) Run(ctx context.Context) error {
 		}
 
 		klog.V(4).Info("initializing Prometheus object")
-		p, err := t.factory.PrometheusK8s(s, trustedCA)
+		p, err := t.factory.PrometheusK8s(s, trustedCA, telemetrySecret)
 		if err != nil {
 			return errors.Wrap(err, "initializing Prometheus object failed")
 		}