From 078c73de8a6b99cb24ad86b8dd7d4bd3e3d90ec9 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Mon, 7 Oct 2024 15:37:14 +0200 Subject: [PATCH] Revert "feat: TLS support for the Thanos web endpoint (#496)" This reverts commit 43ee64ef5ffe951fe47a9a7124034a6d5a44d059. --- .../monitoring.rhobs_thanosqueriers.yaml | 61 ------- .../monitoring.rhobs_thanosqueriers.yaml | 61 ------- docs/api.md | 150 ------------------ pkg/apis/monitoring/v1alpha1/types.go | 3 - .../v1alpha1/zz_generated.deepcopy.go | 5 - .../monitoring/thanos-querier/components.go | 122 +------------- .../monitoring/thanos-querier/controller.go | 123 +------------- test/e2e/thanos_querier_controller_test.go | 113 ------------- 8 files changed, 8 insertions(+), 630 deletions(-) diff --git a/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml b/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml index c41580a13..cc5275d0e 100644 --- a/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml +++ b/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml @@ -110,67 +110,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - webTLSConfig: - description: Configure TLS options for the Thanos web server. - properties: - certificate: - description: Reference to the TLS public certificate for the web - server. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - minLength: 1 - type: string - name: - description: The name of the secret in the object's namespace - to select from. - minLength: 1 - type: string - required: - - key - - name - type: object - certificateAuthority: - description: Reference to the root Certificate Authority used - to verify the web server's certificate. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - minLength: 1 - type: string - name: - description: The name of the secret in the object's namespace - to select from. - minLength: 1 - type: string - required: - - key - - name - type: object - privateKey: - description: Reference to the TLS private key for the web server. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - minLength: 1 - type: string - name: - description: The name of the secret in the object's namespace - to select from. - minLength: 1 - type: string - required: - - key - - name - type: object - required: - - certificate - - certificateAuthority - - privateKey - type: object required: - selector type: object diff --git a/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml b/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml index e64f4ded5..d708ea933 100644 --- a/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml +++ b/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml @@ -110,67 +110,6 @@ spec: type: object type: object x-kubernetes-map-type: atomic - webTLSConfig: - description: Configure TLS options for the Thanos web server. - properties: - certificate: - description: Reference to the TLS public certificate for the web - server. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - minLength: 1 - type: string - name: - description: The name of the secret in the object's namespace - to select from. - minLength: 1 - type: string - required: - - key - - name - type: object - certificateAuthority: - description: Reference to the root Certificate Authority used - to verify the web server's certificate. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - minLength: 1 - type: string - name: - description: The name of the secret in the object's namespace - to select from. - minLength: 1 - type: string - required: - - key - - name - type: object - privateKey: - description: Reference to the TLS private key for the web server. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - minLength: 1 - type: string - name: - description: The name of the secret in the object's namespace - to select from. - minLength: 1 - type: string - required: - - key - - name - type: object - required: - - certificate - - certificateAuthority - - privateKey - type: object required: - selector type: object diff --git a/docs/api.md b/docs/api.md index 51a8419cb..9b51d3587 100644 --- a/docs/api.md +++ b/docs/api.md @@ -3690,13 +3690,6 @@ deduplicate.
false - - webTLSConfig - object - - Configure TLS options for the Thanos web server.
- - false @@ -3817,149 +3810,6 @@ list restricting them.
- -### ThanosQuerier.spec.webTLSConfig -[↩ Parent](#thanosquerierspec) - - - -Configure TLS options for the Thanos web server. - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionRequired
certificateobject - Reference to the TLS public certificate for the web server.
- 		true
certificateAuthorityobject - Reference to the root Certificate Authority used to verify the web server's certificate.
- 		true
privateKeyobject - Reference to the TLS private key for the web server.
- 		true
- - -### ThanosQuerier.spec.webTLSConfig.certificate -[↩ Parent](#thanosquerierspecwebtlsconfig) - - - -Reference to the TLS public certificate for the web server. - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionRequired
keystring - The key of the secret to select from. Must be a valid secret key.
- 		true
namestring - The name of the secret in the object's namespace to select from.
- 		true
- - -### ThanosQuerier.spec.webTLSConfig.certificateAuthority -[↩ Parent](#thanosquerierspecwebtlsconfig) - - - -Reference to the root Certificate Authority used to verify the web server's certificate. - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionRequired
keystring - The key of the secret to select from. Must be a valid secret key.
- 		true
namestring - The name of the secret in the object's namespace to select from.
- 		true
- - -### ThanosQuerier.spec.webTLSConfig.privateKey -[↩ Parent](#thanosquerierspecwebtlsconfig) - - - -Reference to the TLS private key for the web server. - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionRequired
keystring - The key of the secret to select from. Must be a valid secret key.
- 		true
namestring - The name of the secret in the object's namespace to select from.
- 		true
- # observability.openshift.io/v1alpha1 Resource Types: diff --git a/pkg/apis/monitoring/v1alpha1/types.go b/pkg/apis/monitoring/v1alpha1/types.go index 18eb00df6..6ed6205a2 100644 --- a/pkg/apis/monitoring/v1alpha1/types.go +++ b/pkg/apis/monitoring/v1alpha1/types.go @@ -279,9 +279,6 @@ type ThanosQuerierSpec struct { // Selector to select which namespaces the Monitoring Stack objects are discovered from. NamespaceSelector NamespaceSelector `json:"namespaceSelector,omitempty"` ReplicaLabels []string `json:"replicaLabels,omitempty"` - // Configure TLS options for the Thanos web server. - // +optional - WebTLSConfig *WebTLSConfig `json:"webTLSConfig,omitempty"` } // ThanosQuerierStatus defines the observed state of ThanosQuerier. diff --git a/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go index c62674f8d..ca88fb311 100644 --- a/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go @@ -348,11 +348,6 @@ func (in *ThanosQuerierSpec) DeepCopyInto(out *ThanosQuerierSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.WebTLSConfig != nil { - in, out := &in.WebTLSConfig, &out.WebTLSConfig - *out = new(WebTLSConfig) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ThanosQuerierSpec. diff --git a/pkg/controllers/monitoring/thanos-querier/components.go b/pkg/controllers/monitoring/thanos-querier/components.go index 160684a78..905539953 100644 --- a/pkg/controllers/monitoring/thanos-querier/components.go +++ b/pkg/controllers/monitoring/thanos-querier/components.go @@ -13,54 +13,17 @@ import ( "github.com/rhobs/observability-operator/pkg/reconciler" ) -func thanosComponentReconcilers( - thanos *msoapi.ThanosQuerier, - sidecarUrls []string, - thanosCfg ThanosConfiguration, - tlsHashes map[string]string, -) []reconciler.Reconciler { +func thanosComponentReconcilers(thanos *msoapi.ThanosQuerier, sidecarUrls []string, thanosCfg ThanosConfiguration) []reconciler.Reconciler { name := "thanos-querier-" + thanos.Name return []reconciler.Reconciler{ reconciler.NewUpdater(newServiceAccount(name, thanos.Namespace), thanos), - reconciler.NewUpdater(newThanosQuerierDeployment(name, thanos, sidecarUrls, thanosCfg, tlsHashes), thanos), + reconciler.NewUpdater(newThanosQuerierDeployment(name, thanos, sidecarUrls, thanosCfg), thanos), reconciler.NewUpdater(newService(name, thanos.Namespace), thanos), - reconciler.NewUpdater(newServiceMonitor(name, thanos.Namespace, thanos), thanos), - reconciler.NewOptionalUpdater(newHttpConfConfigMap(name, thanos), thanos, thanos.Spec.WebTLSConfig != nil), + reconciler.NewUpdater(newServiceMonitor(name, thanos.Namespace), thanos), } } -func newHttpConfConfigMap(name string, thanos *msoapi.ThanosQuerier) *corev1.ConfigMap { - httpConf := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-http-conf", name), - Namespace: thanos.Namespace, - }, - } - if thanos.Spec.WebTLSConfig != nil { - httpConf.Data = map[string]string{ - "http.conf": ` -tls_server_config: - cert_file: /etc/thanos/tls-assets/web-cert-secret/` + thanos.Spec.WebTLSConfig.Certificate.Key + ` - key_file: /etc/thanos/tls-assets/web-key-secret/` + thanos.Spec.WebTLSConfig.PrivateKey.Key, - } - } - - return httpConf -} - -func newThanosQuerierDeployment( - name string, - spec *msoapi.ThanosQuerier, - sidecarUrls []string, - thanosCfg ThanosConfiguration, - tlsHashes map[string]string, -) *appsv1.Deployment { - httpConfCMName := fmt.Sprintf("%s-http-conf", name) - +func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecarUrls []string, thanosCfg ThanosConfiguration) *appsv1.Deployment { args := []string{ "query", "--log.format=logfmt", @@ -75,10 +38,6 @@ func newThanosQuerierDeployment( args = append(args, fmt.Sprintf("--query.replica-label=%s", rl)) } - if spec.Spec.WebTLSConfig != nil { - args = append(args, "--http.config=/etc/thanos/tls-assets/web-http-conf-cm/http.conf") - } - thanos := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: appsv1.SchemeGroupVersion.String(), @@ -143,58 +102,6 @@ func newThanosQuerierDeployment( ProgressDeadlineSeconds: ptr.To(int32(300)), }, } - if spec.Spec.WebTLSConfig != nil { - thanos.Spec.Template.Spec.Volumes = append(thanos.Spec.Template.Spec.Volumes, []corev1.Volume{ - { - Name: "thanos-web-tls-key", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: spec.Spec.WebTLSConfig.PrivateKey.Name, - }, - }, - }, - { - Name: "thanos-web-tls-cert", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: spec.Spec.WebTLSConfig.Certificate.Name, - }, - }, - }, - { - Name: "thanos-web-http-conf", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: httpConfCMName, - }, - }, - }, - }, - }...) - thanos.Spec.Template.Spec.Containers[0].VolumeMounts = append(thanos.Spec.Template.Spec.Containers[0].VolumeMounts, []corev1.VolumeMount{ - { - Name: "thanos-web-tls-key", - MountPath: "/etc/thanos/tls-assets/web-cert-secret", - ReadOnly: true, - }, - { - Name: "thanos-web-tls-cert", - MountPath: "/etc/thanos/tls-assets/web-key-secret", - ReadOnly: true, - }, - { - Name: "thanos-web-http-conf", - MountPath: "/etc/thanos/tls-assets/web-http-conf-cm", - ReadOnly: true, - }, - }...) - tlsAnnotations := map[string]string{} - for name, hash := range tlsHashes { - tlsAnnotations[fmt.Sprintf("monitoring.openshift.io/%s-hash", name)] = hash - } - thanos.ObjectMeta.Annotations = tlsAnnotations - } return thanos } @@ -237,8 +144,8 @@ func newService(name string, namespace string) *corev1.Service { } } -func newServiceMonitor(name string, namespace string, thanos *msoapi.ThanosQuerier) *monv1.ServiceMonitor { - serviceMonitor := &monv1.ServiceMonitor{ +func newServiceMonitor(name string, namespace string) *monv1.ServiceMonitor { + return &monv1.ServiceMonitor{ TypeMeta: metav1.TypeMeta{ APIVersion: monv1.SchemeGroupVersion.String(), Kind: "ServiceMonitor", @@ -262,23 +169,6 @@ func newServiceMonitor(name string, namespace string, thanos *msoapi.ThanosQueri }, }, } - if thanos.Spec.WebTLSConfig != nil { - serviceMonitor.Spec.Endpoints[0].Scheme = "https" - serviceMonitor.Spec.Endpoints[0].TLSConfig = &monv1.TLSConfig{ - SafeTLSConfig: monv1.SafeTLSConfig{ - CA: monv1.SecretOrConfigMap{ - Secret: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: thanos.Spec.WebTLSConfig.CertificateAuthority.Name, - }, - Key: thanos.Spec.WebTLSConfig.CertificateAuthority.Key, - }, - }, - ServerName: ptr.To(name), - }, - } - } - return serviceMonitor } func componentLabels(querierName string) map[string]string { diff --git a/pkg/controllers/monitoring/thanos-querier/controller.go b/pkg/controllers/monitoring/thanos-querier/controller.go index b9285fe72..87275a996 100644 --- a/pkg/controllers/monitoring/thanos-querier/controller.go +++ b/pkg/controllers/monitoring/thanos-querier/controller.go @@ -14,7 +14,6 @@ package thanos_querier import ( "context" - "crypto/sha256" "fmt" "time" @@ -23,11 +22,9 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/rand" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,12 +51,6 @@ type Options struct { Thanos ThanosConfiguration } -const ( - thanosTLSPrivateKeySecretNameField = ".spec.webTLSConfig.privateKey.name" - thanosTLSCertificateSecretNameField = ".spec.webTLSConfig.certificate.name" - thanosTLSCertificateAuthoritySecretNameField = ".spec.webTLSConfig.certificateAuthority.name" -) - // RBAC for watching monitoring stacks //+kubebuilder:rbac:groups=monitoring.rhobs,resources=monitoringstacks,verbs=list;watch @@ -72,8 +63,7 @@ const ( //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=list;watch;create;update;patch;delete // RBAC for managing core resources -//+kubebuilder:rbac:groups=core,resources=services;serviceaccounts;configmaps,verbs=list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=core,resources=secrets,verbs=list;watch +//+kubebuilder:rbac:groups=core,resources=services;serviceaccounts,verbs=list;watch;create;update;patch;delete // RBAC for managing Prometheus Operator CRs //+kubebuilder:rbac:groups=monitoring.rhobs,resources=servicemonitors,verbs=list;watch;create;update;patch;delete @@ -89,56 +79,17 @@ func RegisterWithManager(mgr ctrl.Manager, opts Options) error { thanos: opts.Thanos, } - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSPrivateKeySecretNameField, func(rawObj client.Object) []string { - // Extract the secret name from the spec, if one is provided - cr := rawObj.(*msoapi.ThanosQuerier) - if cr.Spec.WebTLSConfig == nil { - return nil - } - return []string{cr.Spec.WebTLSConfig.PrivateKey.Name} - }); err != nil { - return err - } - - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSCertificateSecretNameField, func(rawObj client.Object) []string { - // Extract the secret name from the spec, if one is provided - cr := rawObj.(*msoapi.ThanosQuerier) - if cr.Spec.WebTLSConfig == nil { - return nil - } - return []string{cr.Spec.WebTLSConfig.Certificate.Name} - }); err != nil { - return err - } - - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSCertificateAuthoritySecretNameField, func(rawObj client.Object) []string { - // Extract the secret name from the spec, if one is provided - cr := rawObj.(*msoapi.ThanosQuerier) - if cr.Spec.WebTLSConfig == nil { - return nil - } - return []string{cr.Spec.WebTLSConfig.CertificateAuthority.Name} - }); err != nil { - return err - } - p := predicate.GenerationChangedPredicate{} return ctrl.NewControllerManagedBy(mgr). For(&msoapi.ThanosQuerier{}). Owns(&appsv1.Deployment{}).WithEventFilter(p). Owns(&corev1.ServiceAccount{}).WithEventFilter(p). Owns(&corev1.Service{}).WithEventFilter(p). - Owns(&corev1.ConfigMap{}).WithEventFilter(p). Watches( &msoapi.MonitoringStack{}, handler.EnqueueRequestsFromMapFunc(rm.findQueriersForMonitoringStack), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). - Watches( - &corev1.Secret{}, - handler.EnqueueRequestsFromMapFunc(rm.findQueriersForTLSSecrets), - builder.WithPredicates(predicate.GenerationChangedPredicate{}), - ). Complete(rm) } @@ -162,23 +113,7 @@ func (rm resourceManager) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{RequeueAfter: 10 * time.Second}, err } - tlsHashes := map[string]string{} - if querier.Spec.WebTLSConfig != nil { - secretSelectors := []msoapi.SecretKeySelector{ - querier.Spec.WebTLSConfig.CertificateAuthority, - querier.Spec.WebTLSConfig.Certificate, - querier.Spec.WebTLSConfig.PrivateKey, - } - for _, secretSelector := range secretSelectors { - hash, err := rm.hashOfTLSSecret(secretSelector, querier.Namespace) - if err != nil { - return ctrl.Result{}, err - } - tlsHashes[fmt.Sprintf("%s-%s", secretSelector.Name, secretSelector.Key)] = hash - } - } - - reconcilers := thanosComponentReconcilers(querier, sidecarServices, rm.thanos, tlsHashes) + reconcilers := thanosComponentReconcilers(querier, sidecarServices, rm.thanos) for _, reconciler := range reconcilers { err := reconciler.Reconcile(ctx, rm, rm.scheme) // handle creation / updation errors that can happen due to a stale cache by @@ -221,20 +156,6 @@ func (rm resourceManager) findSidecarServices(ctx context.Context, tQuerier *mso return sidecarUrls, nil } -func (rm resourceManager) hashOfTLSSecret(selector msoapi.SecretKeySelector, namespace string) (string, error) { - var secret corev1.Secret - err := rm.Get(context.Background(), types.NamespacedName{ - Name: selector.Name, - Namespace: namespace, - }, &secret) - if err != nil { - return "", fmt.Errorf("Couldn't get TLS secret %s: %s", selector.Name, err) - } - - hash := sha256.Sum256(secret.Data[selector.Key]) - return rand.SafeEncodeString(fmt.Sprint(hash)), nil -} - // Given a Service object, return a url to use as value for --store/--endpoint. func getEndpointUrl(serviceName string, namespace string) string { return fmt.Sprintf("dnssrv+_grpc._tcp.%s.%s.svc.cluster.local", serviceName, namespace) @@ -270,43 +191,3 @@ func (rm resourceManager) findQueriersForMonitoringStack(ctx context.Context, ms } return requests } - -// Find all ThanosQueriers, whose TLS secrets fit the given Secret and -// return a list of reconcile requests, one for each ThanosQuerier. -func (rm resourceManager) findQueriersForTLSSecrets(ctx context.Context, src client.Object) []reconcile.Request { - requests := []reconcile.Request{} - - logger := rm.logger.WithValues("Secret", src.GetNamespace()+"/"+src.GetName()) - logger.Info("watched Secret changed, checking for matching querier") - - thanosWatchFields := []string{ - thanosTLSCertificateAuthoritySecretNameField, - thanosTLSCertificateSecretNameField, - thanosTLSPrivateKeySecretNameField, - } - - for _, field := range thanosWatchFields { - crList := &msoapi.ThanosQuerierList{} - listOps := &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), - Namespace: src.GetNamespace(), - } - err := rm.Client.List(ctx, crList, listOps) - if err != nil { - logger.Error(err, "Failed to list Thanosqueriers") - return []reconcile.Request{} - } - - for _, item := range crList.Items { - logger.Info("Found querier, scheduling sync") - requests = append(requests, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: item.GetName(), - Namespace: item.GetNamespace(), - }, - }) - } - } - - return requests -} diff --git a/test/e2e/thanos_querier_controller_test.go b/test/e2e/thanos_querier_controller_test.go index b5a5c3502..f42255e09 100644 --- a/test/e2e/thanos_querier_controller_test.go +++ b/test/e2e/thanos_querier_controller_test.go @@ -3,8 +3,6 @@ package e2e import ( "context" "fmt" - "net" - "strings" "testing" "time" @@ -15,7 +13,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/cert" "sigs.k8s.io/controller-runtime/pkg/client" msov1 "github.com/rhobs/observability-operator/pkg/apis/monitoring/v1alpha1" @@ -34,10 +31,6 @@ func TestThanosQuerierController(t *testing.T) { name: "Delete resources if matched monitoring stack is deleted", scenario: stackWithSidecarGetsDeleted, }, - { - name: "Create resources for single monitoring stack with web endpoint TLS", - scenario: singleStackWithSidecarTLS, - }, } for _, tc := range ts { @@ -122,112 +115,6 @@ func singleStackWithSidecar(t *testing.T) { } } -func singleStackWithSidecarTLS(t *testing.T) { - comboName := "tq-ms-combo-tls" - querierName := "thanos-querier-" + comboName - - certs, key, err := cert.GenerateSelfSignedCertKey(querierName, []net.IP{}, []string{}) - assert.NilError(t, err) - - thanosKey := string(key) - thanosCerts := strings.SplitAfter(string(certs), "-----END CERTIFICATE-----") - - tlsSecretName := "thanos-test-tls-secret" - - thanosTLSSecret := corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: tlsSecretName, - Namespace: e2eTestNamespace, - }, - StringData: map[string]string{ - "tls.key": thanosKey, - "tls.crt": thanosCerts[0], - "ca.crt": thanosCerts[1], - }, - } - err = f.K8sClient.Create(context.Background(), &thanosTLSSecret) - assert.NilError(t, err) - - tq, ms := newThanosStackCombo(t, comboName) - tq.Spec.WebTLSConfig = &msov1.WebTLSConfig{ - PrivateKey: msov1.SecretKeySelector{ - Name: tlsSecretName, - Key: "tls.key", - }, - Certificate: msov1.SecretKeySelector{ - Name: tlsSecretName, - Key: "tls.crt", - }, - CertificateAuthority: msov1.SecretKeySelector{ - Name: tlsSecretName, - Key: "ca.crt", - }, - } - err = f.K8sClient.Create(context.Background(), tq) - assert.NilError(t, err, "failed to create a thanos querier") - err = f.K8sClient.Create(context.Background(), ms) - assert.NilError(t, err, "failed to create a monitoring stack") - - // Creating a basic combo must create a thanos deployment and a service - thanosDeployment := appsv1.Deployment{} - f.GetResourceWithRetry(t, querierName, tq.Namespace, &thanosDeployment) - - thanosService := corev1.Service{} - f.GetResourceWithRetry(t, querierName, tq.Namespace, &thanosService) - - f.AssertDeploymentReady(querierName, tq.Namespace, framework.WithTimeout(5*time.Minute))(t) - // Assert prometheus instance can be queried - stopChan := make(chan struct{}) - defer close(stopChan) - if err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { - err = f.StartServicePortForward(querierName, e2eTestNamespace, "10902", stopChan) - return err == nil, nil - }); wait.Interrupted(err) { - t.Fatal("timeout waiting for port-forward") - } - - promClient, err := framework.NewTLSPrometheusClient("https://localhost:10902", thanosCerts[1], querierName) - if err != nil { - t.Fatal(fmt.Errorf("Failed to create prometheus client: %s", err)) - } - expectedResults := map[string]int{ - "prometheus_build_info": 2, // must return from both prometheus pods - } - var lastErr error - if err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { - correct := 0 - for query, value := range expectedResults { - result, err := promClient.Query(query) - if err != nil { - return false, nil - } - - if len(result.Data.Result) == 0 { - return false, nil - } - - if len(result.Data.Result) > value { - lastErr = fmt.Errorf("invalid result for query %s, got %d, want %d", query, len(result.Data.Result), value) - return true, lastErr - } - - if len(result.Data.Result) != value { - return false, nil - } - - correct++ - } - - return correct == len(expectedResults), nil - }); wait.Interrupted(err) { - t.Fatal(fmt.Errorf("querying thanos did not yield expected results: %w", lastErr)) - } -} - func newThanosQuerier(t *testing.T, name string, selector map[string]string) *msov1.ThanosQuerier { tq := &msov1.ThanosQuerier{ ObjectMeta: metav1.ObjectMeta{