diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index bfdf90603f6bf..8bc466b69bf31 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,5 +1,6 @@ ## Main +- [6288](https://github.com/grafana/loki/pull/6288) **aminesnow**: Expose only an HTTPS gateway when in openshift mode - [6195](https://github.com/grafana/loki/pull/6195) **periklis**: Add ruler config support - [6198](https://github.com/grafana/loki/pull/6198) **periklis**: Add support for custom S3 CA - [6199](https://github.com/grafana/loki/pull/6199) **Red-GV**: Update GCP secret volume path diff --git a/operator/hack/addons_ocp.yaml b/operator/hack/addons_ocp.yaml index 32d43f7339ea1..312deee33af08 100644 --- a/operator/hack/addons_ocp.yaml +++ b/operator/hack/addons_ocp.yaml @@ -37,12 +37,12 @@ spec: - name: LOKI_ORG_ID value: application - name: LOKI_ADDR - value: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application + value: https://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application - name: LOKI_BEARER_TOKEN_FILE value: /var/run/secrets/kubernetes.io/serviceaccount/token args: - -c - - while true; do logcli query '{job="systemd-journal"}'; sleep 30; done + - while true; do logcli --ca-cert="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" query '{job="systemd-journal"}'; sleep 30; done serviceAccountName: lokistack-dev-addons-logcli --- apiVersion: apps/v1 @@ -118,7 +118,9 @@ metadata: data: promtail.yaml: | clients: - - url: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application/loki/api/v1/push + - url: https://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application/loki/api/v1/push + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt tenant_id: application bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token backoff_config: diff --git a/operator/hack/lokistack_gateway_ocp.yaml b/operator/hack/lokistack_gateway_ocp.yaml index 2d5e4e949c3a1..a46c1e22db26b 100644 --- a/operator/hack/lokistack_gateway_ocp.yaml +++ b/operator/hack/lokistack_gateway_ocp.yaml @@ -2,6 +2,7 @@ apiVersion: loki.grafana.com/v1beta1 kind: LokiStack metadata: name: lokistack-dev + namespace: openshift-logging spec: size: 1x.extra-small storage: diff --git a/operator/internal/manifests/build_test.go b/operator/internal/manifests/build_test.go index c72c152fb60d7..3e03e9f92ec6a 100644 --- a/operator/internal/manifests/build_test.go +++ b/operator/internal/manifests/build_test.go @@ -263,7 +263,7 @@ func TestBuildAll_WithFeatureFlags_EnableTLSServiceMonitorConfig(t *testing.T) { continue } - secretName := fmt.Sprintf("%s-http-metrics", name) + secretName := fmt.Sprintf("%s-http-tls", name) expVolume := corev1.Volume{ Name: secretName, VolumeSource: corev1.VolumeSource{ diff --git a/operator/internal/manifests/gateway.go b/operator/internal/manifests/gateway.go index f617eaa51f367..535ee41a2451a 100644 --- a/operator/internal/manifests/gateway.go +++ b/operator/internal/manifests/gateway.go @@ -20,7 +20,7 @@ import ( ) const ( - tlsMetricsSercetVolume = "tls-metrics-secret" + tlsSecretVolume = "tls-secret" ) // BuildGateway returns a list of k8s objects for Loki Stack Gateway @@ -49,7 +49,7 @@ func BuildGateway(opts Options) ([]client.Object, error) { if opts.Stack.Tenants != nil { mode := opts.Stack.Tenants.Mode - if err := configureDeploymentForMode(dpl, mode, opts.Flags); err != nil { + if err := configureDeploymentForMode(dpl, mode, opts.Flags, opts.Name, opts.Namespace); err != nil { return nil, err } @@ -356,7 +356,7 @@ func configureGatewayMetricsPKI(podSpec *corev1.PodSpec, serviceName string) err secretVolumeSpec := corev1.PodSpec{ Volumes: []corev1.Volume{ { - Name: tlsMetricsSercetVolume, + Name: tlsSecretVolume, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: secretName, @@ -368,7 +368,7 @@ func configureGatewayMetricsPKI(podSpec *corev1.PodSpec, serviceName string) err secretContainerSpec := corev1.Container{ VolumeMounts: []corev1.VolumeMount{ { - Name: tlsMetricsSercetVolume, + Name: tlsSecretVolume, ReadOnly: true, MountPath: gateway.LokiGatewayTLSDir, }, diff --git a/operator/internal/manifests/gateway_tenants.go b/operator/internal/manifests/gateway_tenants.go index edf2b63e1ab49..31737719a341c 100644 --- a/operator/internal/manifests/gateway_tenants.go +++ b/operator/internal/manifests/gateway_tenants.go @@ -56,15 +56,18 @@ func ApplyGatewayDefaultOptions(opts *Options) error { return nil } -func configureDeploymentForMode(d *appsv1.Deployment, mode lokiv1beta1.ModeType, flags FeatureFlags) error { +func configureDeploymentForMode(d *appsv1.Deployment, mode lokiv1beta1.ModeType, flags FeatureFlags, stackName, stackNs string) error { switch mode { case lokiv1beta1.Static, lokiv1beta1.Dynamic: return nil // nothing to configure case lokiv1beta1.OpenshiftLogging: + serviceName := serviceNameGatewayHTTP(stackName) + secretName := signingServiceSecretName(serviceName) + serverName := fqdn(serviceName, stackNs) return openshift.ConfigureGatewayDeployment( d, gatewayContainerName, - tlsMetricsSercetVolume, + tlsSecretVolume, gateway.LokiGatewayTLSDir, gateway.LokiGatewayCertFile, gateway.LokiGatewayKeyFile, @@ -72,6 +75,9 @@ func configureDeploymentForMode(d *appsv1.Deployment, mode lokiv1beta1.ModeType, gateway.LokiGatewayCAFile, flags.EnableTLSServiceMonitorConfig, flags.EnableCertificateSigningService, + secretName, + serverName, + gatewayHTTPPort, ) } diff --git a/operator/internal/manifests/gateway_tenants_test.go b/operator/internal/manifests/gateway_tenants_test.go index 3e106bc9046a0..c6ed473802d74 100644 --- a/operator/internal/manifests/gateway_tenants_test.go +++ b/operator/internal/manifests/gateway_tenants_test.go @@ -1,6 +1,7 @@ package manifests import ( + "fmt" "testing" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -178,11 +179,13 @@ func TestApplyGatewayDefaultsOptions(t *testing.T) { func TestConfigureDeploymentForMode(t *testing.T) { type tt struct { - desc string - mode lokiv1beta1.ModeType - flags FeatureFlags - dpl *appsv1.Deployment - want *appsv1.Deployment + desc string + mode lokiv1beta1.ModeType + stackName string + stackNs string + flags FeatureFlags + dpl *appsv1.Deployment + want *appsv1.Deployment } tc := []tt{ @@ -199,9 +202,15 @@ func TestConfigureDeploymentForMode(t *testing.T) { want: &appsv1.Deployment{}, }, { - desc: "openshift-logging mode", - mode: lokiv1beta1.OpenshiftLogging, + + desc: "openshift-logging mode", + mode: lokiv1beta1.OpenshiftLogging, + stackName: "test", + stackNs: "test-ns", dpl: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ @@ -212,6 +221,21 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--logs.read.endpoint=http://example.com", "--logs.tail.endpoint=http://example.com", "--logs.write.endpoint=http://example.com", + fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + }, + }, }, }, }, @@ -220,6 +244,9 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, }, want: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ @@ -230,6 +257,32 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--logs.read.endpoint=http://example.com", "--logs.tail.endpoint=http://example.com", "--logs.write.endpoint=http://example.com", + fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), + "--tls.server.cert-file=/var/run/tls/tls.crt", + "--tls.server.key-file=/var/run/tls/tls.key", + "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", + fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: tlsSecretVolume, + ReadOnly: true, + MountPath: gateway.LokiGatewayTLSDir, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, + }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, + }, + }, }, }, { @@ -284,18 +337,33 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, }, }, + Volumes: []corev1.Volume{ + { + Name: tlsSecretVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test-gateway-http-tls", + }, + }, + }, + }, }, }, }, }, }, { - desc: "openshift-logging mode with-tls-service-monitor-config", - mode: lokiv1beta1.OpenshiftLogging, + desc: "openshift-logging mode with-tls-service-monitor-config", + mode: lokiv1beta1.OpenshiftLogging, + stackName: "test", + stackNs: "test-ns", flags: FeatureFlags{ EnableTLSServiceMonitorConfig: true, }, dpl: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ @@ -306,6 +374,38 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--logs.read.endpoint=http://example.com", "--logs.tail.endpoint=http://example.com", "--logs.write.endpoint=http://example.com", + fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: tlsSecretVolume, + ReadOnly: true, + MountPath: gateway.LokiGatewayTLSDir, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: tlsSecretVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test-gateway-http-tls", + }, }, }, }, @@ -314,6 +414,9 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, }, want: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ @@ -324,6 +427,32 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--logs.read.endpoint=http://example.com", "--logs.tail.endpoint=http://example.com", "--logs.write.endpoint=http://example.com", + fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), + "--tls.server.cert-file=/var/run/tls/tls.crt", + "--tls.server.key-file=/var/run/tls/tls.key", + "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", + fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: tlsSecretVolume, + ReadOnly: true, + MountPath: gateway.LokiGatewayTLSDir, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, + }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, + }, + }, }, }, { @@ -380,28 +509,41 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, VolumeMounts: []corev1.VolumeMount{ { - Name: tlsMetricsSercetVolume, + Name: tlsSecretVolume, ReadOnly: true, MountPath: gateway.LokiGatewayTLSDir, }, }, }, }, + Volumes: []corev1.Volume{ + { + Name: tlsSecretVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test-gateway-http-tls", + }, + }, + }, + }, }, }, }, }, }, { - desc: "openshift-logging mode with-cert-signing-service", - mode: lokiv1beta1.OpenshiftLogging, + desc: "openshift-logging mode with-cert-signing-service", + mode: lokiv1beta1.OpenshiftLogging, + stackName: "test", + stackNs: "test-ns", flags: FeatureFlags{ EnableTLSServiceMonitorConfig: true, EnableCertificateSigningService: true, }, dpl: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "gateway", + Name: "gateway", + Namespace: "test-ns", }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -414,6 +556,7 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--logs.read.endpoint=http://example.com", "--logs.tail.endpoint=http://example.com", "--logs.write.endpoint=http://example.com", + fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), }, VolumeMounts: []corev1.VolumeMount{ { @@ -422,6 +565,20 @@ func TestConfigureDeploymentForMode(t *testing.T) { MountPath: "/var/run/tls", }, }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + }, + }, + }, }, }, Volumes: []corev1.Volume{ @@ -435,7 +592,8 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, want: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "gateway", + Name: "gateway", + Namespace: "test-ns", }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -449,7 +607,12 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--logs.read.endpoint=https://example.com", "--logs.tail.endpoint=https://example.com", "--logs.write.endpoint=https://example.com", + fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), "--logs.tls.ca-file=/var/run/ca/service-ca.crt", + "--tls.server.cert-file=/var/run/tls/tls.crt", + "--tls.server.key-file=/var/run/tls/tls.key", + "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", + fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), }, VolumeMounts: []corev1.VolumeMount{ { @@ -463,6 +626,20 @@ func TestConfigureDeploymentForMode(t *testing.T) { MountPath: "/var/run/ca", }, }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, + }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, + }, + }, + }, }, { Name: "opa", @@ -518,7 +695,7 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, VolumeMounts: []corev1.VolumeMount{ { - Name: tlsMetricsSercetVolume, + Name: tlsSecretVolume, ReadOnly: true, MountPath: gateway.LokiGatewayTLSDir, }, @@ -551,7 +728,7 @@ func TestConfigureDeploymentForMode(t *testing.T) { tc := tc t.Run(tc.desc, func(t *testing.T) { t.Parallel() - err := configureDeploymentForMode(tc.dpl, tc.mode, tc.flags) + err := configureDeploymentForMode(tc.dpl, tc.mode, tc.flags, "test", "test-ns") require.NoError(t, err) require.Equal(t, tc.want, tc.dpl) }) diff --git a/operator/internal/manifests/openshift/configure.go b/operator/internal/manifests/openshift/configure.go index 0379648fc2b46..a66403c324c2b 100644 --- a/operator/internal/manifests/openshift/configure.go +++ b/operator/internal/manifests/openshift/configure.go @@ -2,6 +2,7 @@ package openshift import ( "fmt" + "path" "regexp" "strings" @@ -36,12 +37,15 @@ var ( // ConfigureGatewayDeployment merges an OpenPolicyAgent sidecar into the deployment spec. // With this, the deployment will route authorization request to the OpenShift // apiserver through the sidecar. +// This function also forces the use of a TLS connection for the gateway. func ConfigureGatewayDeployment( d *appsv1.Deployment, gwContainerName string, - sercretVolumeName, tlsDir, certFile, keyFile string, + secretVolumeName, tlsDir, certFile, keyFile string, caDir, caFile string, withTLS, withCertSigningService bool, + secretName, serverName string, + gatewayHTTPPort int, ) error { var gwIndex int for i, c := range d.Spec.Template.Spec.Containers { @@ -89,13 +93,49 @@ func ConfigureGatewayDeployment( }) } + for i, a := range gwArgs { + if strings.HasPrefix(a, "--web.healthchecks.url=") { + gwArgs[i] = fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort) + break + } + } + + certFilePath := path.Join(tlsDir, certFile) + keyFilePath := path.Join(tlsDir, keyFile) + caFilePath := path.Join(caDir, caFile) + gwArgs = append(gwArgs, + fmt.Sprintf("--tls.server.cert-file=%s", certFilePath), + fmt.Sprintf("--tls.server.key-file=%s", keyFilePath), + fmt.Sprintf("--tls.healthchecks.server-ca-file=%s", caFilePath), + fmt.Sprintf("--tls.healthchecks.server-name=%s", serverName)) + + gwContainer.ReadinessProbe.ProbeHandler.HTTPGet.Scheme = corev1.URISchemeHTTPS + gwContainer.LivenessProbe.ProbeHandler.HTTPGet.Scheme = corev1.URISchemeHTTPS gwContainer.Args = gwArgs + // Create and mount TLS secrets volumes if it's not already done by the service monitor config. + if !withTLS { + gwVolumes = append(gwVolumes, corev1.Volume{ + Name: secretVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + }, + }, + }) + + gwContainer.VolumeMounts = append(gwContainer.VolumeMounts, corev1.VolumeMount{ + Name: secretVolumeName, + ReadOnly: true, + MountPath: tlsDir, + }) + } + p := corev1.PodSpec{ ServiceAccountName: d.GetName(), Containers: []corev1.Container{ *gwContainer, - newOPAOpenShiftContainer(sercretVolumeName, tlsDir, certFile, keyFile, withTLS), + newOPAOpenShiftContainer(secretVolumeName, tlsDir, certFile, keyFile, withTLS), }, Volumes: gwVolumes, } diff --git a/operator/internal/manifests/openshift/opa_openshift.go b/operator/internal/manifests/openshift/opa_openshift.go index 0fbb26e385d58..000c5b729bedf 100644 --- a/operator/internal/manifests/openshift/opa_openshift.go +++ b/operator/internal/manifests/openshift/opa_openshift.go @@ -19,7 +19,7 @@ const ( opaDefaultLabelMatcher = "kubernetes_namespace_name" ) -func newOPAOpenShiftContainer(sercretVolumeName, tlsDir, certFile, keyFile string, withTLS bool) corev1.Container { +func newOPAOpenShiftContainer(secretVolumeName, tlsDir, certFile, keyFile string, withTLS bool) corev1.Container { var ( image string args []string @@ -55,7 +55,7 @@ func newOPAOpenShiftContainer(sercretVolumeName, tlsDir, certFile, keyFile strin volumeMounts = []corev1.VolumeMount{ { - Name: sercretVolumeName, + Name: secretVolumeName, ReadOnly: true, MountPath: tlsDir, }, diff --git a/operator/internal/manifests/openshift/route.go b/operator/internal/manifests/openshift/route.go index afc4d40fddfc2..18a9f12d71b84 100644 --- a/operator/internal/manifests/openshift/route.go +++ b/operator/internal/manifests/openshift/route.go @@ -29,6 +29,9 @@ func BuildRoute(opts Options) client.Object { Port: &routev1.RoutePort{ TargetPort: intstr.FromString(opts.BuildOpts.GatewaySvcTargetPort), }, + TLS: &routev1.TLSConfig{ + Termination: routev1.TLSTerminationReencrypt, + }, WildcardPolicy: routev1.WildcardPolicyNone, }, } diff --git a/operator/internal/manifests/var.go b/operator/internal/manifests/var.go index f1f1ba72a9b87..0c2747e6b796d 100644 --- a/operator/internal/manifests/var.go +++ b/operator/internal/manifests/var.go @@ -223,7 +223,7 @@ func serviceMonitorName(componentName string) string { } func signingServiceSecretName(serviceName string) string { - return fmt.Sprintf("%s-metrics", serviceName) + return fmt.Sprintf("%s-tls", serviceName) } func fqdn(serviceName, namespace string) string {