diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index d86a2754873d3..172368497bce8 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,3 +1,4 @@ ## Main - [4975](https://github.com/grafana/loki/pull/4975) **periklis**: Provide saner default for loki-operator managed chunk_target_size +- [4974](https://github.com/grafana/loki/pull/5432) **Red-GV**: Provide storage configuration for Azure, GCS, and Swift through common_config diff --git a/operator/api/v1beta1/lokistack_types.go b/operator/api/v1beta1/lokistack_types.go index 681017ec03856..02b53cafc7468 100644 --- a/operator/api/v1beta1/lokistack_types.go +++ b/operator/api/v1beta1/lokistack_types.go @@ -307,13 +307,39 @@ type LokiTemplateSpec struct { IndexGateway *LokiComponentSpec `json:"indexGateway,omitempty"` } +// ObjectStorageSecretType defines the type of storage which can be used with the Loki cluster. +// +// +kubebuilder:validation:Enum=azure;gcs;s3;swift +type ObjectStorageSecretType string + +const ( + // ObjectStorageSecretAzure when using Azure for Loki storage + ObjectStorageSecretAzure ObjectStorageSecretType = "azure" + + // ObjectStorageSecretGCS when using GCS for Loki storage + ObjectStorageSecretGCS ObjectStorageSecretType = "gcs" + + // ObjectStorageSecretS3 when using S3 for Loki storage + ObjectStorageSecretS3 ObjectStorageSecretType = "s3" + + // ObjectStorageSecretSwift when using Swift for Loki storage + ObjectStorageSecretSwift ObjectStorageSecretType = "swift" +) + // ObjectStorageSecretSpec is a secret reference containing name only, no namespace. type ObjectStorageSecretSpec struct { + // Type of object storage that should be used + // + // +required + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:select:azure","urn:alm:descriptor:com.tectonic.ui:select:gcs","urn:alm:descriptor:com.tectonic.ui:select:s3","urn:alm:descriptor:com.tectonic.ui:select:swift"},displayName="Object Storage Secret Type" + Type ObjectStorageSecretType `json:"type"` + // Name of a secret in the namespace configured for object storage secrets. // // +required // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:io.kubernetes:Secret",displayName="Object Storage Secret" + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:io.kubernetes:Secret",displayName="Object Storage Secret Name" Name string `json:"name"` } diff --git a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml index 97b67e2b66bb5..ec9541ca95af0 100644 --- a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml @@ -240,10 +240,18 @@ spec: path: storage - description: Name of a secret in the namespace configured for object storage secrets. - displayName: Object Storage Secret + displayName: Object Storage Secret Name path: storage.secret.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret + - description: Type of object storage that should be used + displayName: Object Storage Secret Type + path: storage.secret.type + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:azure + - urn:alm:descriptor:com.tectonic.ui:select:gcs + - urn:alm:descriptor:com.tectonic.ui:select:s3 + - urn:alm:descriptor:com.tectonic.ui:select:swift - description: Storage class name defines the storage class for ingester/querier PVCs. displayName: Storage Class Name diff --git a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml index 1bb04ccee8e84..852bb493de1ca 100644 --- a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml @@ -220,8 +220,17 @@ spec: description: Name of a secret in the namespace configured for object storage secrets. type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + type: string required: - name + - type type: object required: - secret diff --git a/operator/cmd/loki-broker/main.go b/operator/cmd/loki-broker/main.go index 979193a0bb311..76b4e75beba36 100644 --- a/operator/cmd/loki-broker/main.go +++ b/operator/cmd/loki-broker/main.go @@ -11,6 +11,7 @@ import ( "github.com/ViaQ/logerr/log" "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/manifests" + "github.com/grafana/loki/operator/internal/manifests/storage" "sigs.k8s.io/yaml" ) @@ -21,7 +22,7 @@ type config struct { Image string featureFlags manifests.FeatureFlags - objectStorage manifests.ObjectStorage + objectStorage storage.Options crFilepath string writeToDir string @@ -39,12 +40,14 @@ func (c *config) registerFlags(f *flag.FlagSet) { f.BoolVar(&c.featureFlags.EnableTLSServiceMonitorConfig, "with-tls-service-monitors", false, "Enable TLS endpoint for service monitors.") f.BoolVar(&c.featureFlags.EnableGateway, "with-lokistack-gateway", false, "Enables the manifest creation for the entire lokistack-gateway.") // Object storage options - c.objectStorage = manifests.ObjectStorage{} - f.StringVar(&c.objectStorage.Endpoint, "object-storage.endpoint", "", "The S3 endpoint location.") - f.StringVar(&c.objectStorage.Buckets, "object-storage.buckets", "", "A comma-separated list of S3 buckets.") - f.StringVar(&c.objectStorage.Region, "object-storage.region", "", "An S3 region.") - f.StringVar(&c.objectStorage.AccessKeyID, "object-storage.access-key-id", "", "The access key id for S3.") - f.StringVar(&c.objectStorage.AccessKeySecret, "object-storage.access-key-secret", "", "The access key secret for S3.") + c.objectStorage = storage.Options{ + S3: &storage.S3StorageConfig{}, + } + f.StringVar(&c.objectStorage.S3.Endpoint, "object-storage.s3.endpoint", "", "The S3 endpoint location.") + f.StringVar(&c.objectStorage.S3.Buckets, "object-storage.s3.buckets", "", "A comma-separated list of S3 buckets.") + f.StringVar(&c.objectStorage.S3.Region, "object-storage.s3.region", "", "An S3 region.") + f.StringVar(&c.objectStorage.S3.AccessKeyID, "object-storage.s3.access-key-id", "", "The access key id for S3.") + f.StringVar(&c.objectStorage.S3.AccessKeySecret, "object-storage.s3.access-key-secret", "", "The access key secret for S3.") // Input and output file/dir options f.StringVar(&c.crFilepath, "custom-resource.path", "", "Path to a custom resource YAML file.") f.StringVar(&c.writeToDir, "output.write-dir", "", "write each file to the specified directory.") @@ -64,20 +67,20 @@ func (c *config) validateFlags() { os.Exit(1) } // Validate manifests.objectStorage - if cfg.objectStorage.Endpoint == "" { - log.Info("-object.storage.endpoint flag is required") + if cfg.objectStorage.S3.Endpoint == "" { + log.Info("-object-storage.s3.endpoint flag is required") os.Exit(1) } - if cfg.objectStorage.Buckets == "" { - log.Info("-object.storage.buckets flag is required") + if cfg.objectStorage.S3.Buckets == "" { + log.Info("-object-storage.s3.buckets flag is required") os.Exit(1) } - if cfg.objectStorage.AccessKeyID == "" { - log.Info("-object.storage.access.key.id flag is required") + if cfg.objectStorage.S3.AccessKeyID == "" { + log.Info("-object-storage.s3.access.key.id flag is required") os.Exit(1) } - if cfg.objectStorage.AccessKeySecret == "" { - log.Info("-object.storage.access.key.secret flag is required") + if cfg.objectStorage.S3.AccessKeySecret == "" { + log.Info("-object-storage.s3.access.key.secret flag is required") os.Exit(1) } } diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index fd67db9befb2c..ae454466ee07f 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -173,8 +173,17 @@ spec: name: description: Name of a secret in the namespace configured for object storage secrets. type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + type: string required: - name + - type type: object required: - secret diff --git a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml index 9c6a505b81e36..5628e508a6120 100644 --- a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml @@ -219,10 +219,18 @@ spec: path: storage - description: Name of a secret in the namespace configured for object storage secrets. - displayName: Object Storage Secret + displayName: Object Storage Secret Name path: storage.secret.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret + - description: Type of object storage that should be used + displayName: Object Storage Secret Type + path: storage.secret.type + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:azure + - urn:alm:descriptor:com.tectonic.ui:select:gcs + - urn:alm:descriptor:com.tectonic.ui:select:s3 + - urn:alm:descriptor:com.tectonic.ui:select:swift - description: Storage class name defines the storage class for ingester/querier PVCs. displayName: Storage Class Name diff --git a/operator/docs/hack_operator_make_run.md b/operator/docs/hack_operator_make_run.md index 958cf38fcb491..f024700f2f4dd 100644 --- a/operator/docs/hack_operator_make_run.md +++ b/operator/docs/hack_operator_make_run.md @@ -54,7 +54,7 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ```console kubectl rollout status deployment/ ``` - + where `` is the name of the deployment and can be found using: ```console @@ -62,19 +62,19 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ``` Confirm that all are up and running for `statefulsets` using: - + ```console kubectl rollout status statefulset/ ``` - + where `` is the name of the statefulset and can be found using: - + ```console kubectl get statefulsets ``` - + * If you make some changes to the operator's code, then just stop the operator locally using `CTRL + C`, update the code and rerun the operator locally: - + ```console make run ``` @@ -92,7 +92,7 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ```console make uninstall ``` - + * Cleanup the minio deployment using: ```console @@ -120,7 +120,7 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ```console kubectl get crd lokistacks.loki.grafana.com ``` - + * Create the `openshift-logging` namespace in the cluster: ```console @@ -130,17 +130,15 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev * Now you need to create a storage secret for the operator. This can be done using: ```console - make olm-deploy-example-storage-secret + ./hack/deploy-aws-storage-secret.sh ``` - OR + This secret will be available in `openshift-logging` namespace. You can check the `hack/deploy-aws-storage-secret.sh` file to check the content of the secret. By default, the script will pull credential information using the `aws` cli. However, these values can be overwritten. For example: ```console - ./hack/deploy-example-secret.sh openshift-logging + REGION=us-west-1 ./hack/deploy-aws-storage-secret.sh ``` - This secret will be available in openshift-logging namespace. You can check the `hack/deploy-example-secret.sh` file to check the content of the secret. - * Once the object storage secret is created, you can now create a LokiStack instance: ```console @@ -178,7 +176,7 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ```console kubectl -n openshift-logging get statefulsets ``` - + * If you want `lokistack-gateway` component [1] to be deployed then you need to create a gateway secret [2] for the operator. This can be done using: ```code @@ -187,13 +185,13 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev --from-literal=clientSecret="" \ --from-literal=issuerCAPath="" ``` - + * Now create a LokiStack instance using: ```console kubectl -n openshift-logging apply -f hack/lokistack_gateway_dev.yaml ``` - + * Edit the [main file](https://github.com/grafana/loki/blob/master/operator/main.go) to set the flag values to `true` and rerun the operator using: ```console diff --git a/operator/hack/lokistack_dev.yaml b/operator/hack/lokistack_dev.yaml index 2649ebe9ffa79..568c7d3889675 100644 --- a/operator/hack/lokistack_dev.yaml +++ b/operator/hack/lokistack_dev.yaml @@ -8,4 +8,5 @@ spec: storage: secret: name: test + type: s3 storageClassName: standard diff --git a/operator/hack/lokistack_gateway_dev.yaml b/operator/hack/lokistack_gateway_dev.yaml index aee3cafc6ba53..7f818196ab05d 100644 --- a/operator/hack/lokistack_gateway_dev.yaml +++ b/operator/hack/lokistack_gateway_dev.yaml @@ -8,6 +8,7 @@ spec: storage: secret: name: test + type: s3 storageClassName: gp2 tenants: mode: static diff --git a/operator/hack/lokistack_gateway_ocp.yaml b/operator/hack/lokistack_gateway_ocp.yaml index 8297be48e638c..495980c1405d4 100644 --- a/operator/hack/lokistack_gateway_ocp.yaml +++ b/operator/hack/lokistack_gateway_ocp.yaml @@ -8,6 +8,7 @@ spec: storage: secret: name: test + type: s3 storageClassName: gp2 tenants: mode: openshift-logging diff --git a/operator/internal/handlers/internal/secrets/secrets.go b/operator/internal/handlers/internal/secrets/secrets.go index 88f2f2a4a39cb..898cd18577d23 100644 --- a/operator/internal/handlers/internal/secrets/secrets.go +++ b/operator/internal/handlers/internal/secrets/secrets.go @@ -2,13 +2,101 @@ package secrets import ( "github.com/ViaQ/logerr/kverrors" + lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/manifests" + "github.com/grafana/loki/operator/internal/manifests/storage" corev1 "k8s.io/api/core/v1" ) -// Extract reads a k8s secret into a manifest object storage struct if valid. -func Extract(s *corev1.Secret) (*manifests.ObjectStorage, error) { +// ExtractGatewaySecret reads a k8s secret into a manifest tenant secret struct if valid. +func ExtractGatewaySecret(s *corev1.Secret, tenantName string) (*manifests.TenantSecrets, error) { + // Extract and validate mandatory fields + clientID, ok := s.Data["clientID"] + if !ok { + return nil, kverrors.New("missing clientID field", "field", "clientID") + } + clientSecret, ok := s.Data["clientSecret"] + if !ok { + return nil, kverrors.New("missing clientSecret field", "field", "clientSecret") + } + issuerCAPath, ok := s.Data["issuerCAPath"] + if !ok { + return nil, kverrors.New("missing issuerCAPath field", "field", "issuerCAPath") + } + + return &manifests.TenantSecrets{ + TenantName: tenantName, + ClientID: string(clientID), + ClientSecret: string(clientSecret), + IssuerCAPath: string(issuerCAPath), + }, nil +} + +// ExtractStorageSecret reads a k8s secret into a manifest object storage struct if valid. +func ExtractStorageSecret(s *corev1.Secret, secretType lokiv1beta1.ObjectStorageSecretType) (*storage.Options, error) { + var err error + storage := storage.Options{} + + switch secretType { + case lokiv1beta1.ObjectStorageSecretAzure: + storage.Azure, err = extractAzureConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretGCS: + storage.GCS, err = extractGCSConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretS3: + storage.S3, err = extractS3ConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretSwift: + storage.Swift, err = extractSwiftConfigSecret(s) + default: + return nil, kverrors.New("unknown secret type", "type", secretType) + } + + if err != nil { + return nil, err + } + return &storage, nil +} + +func extractAzureConfigSecret(s *corev1.Secret) (*storage.AzureStorageConfig, error) { + // Extract and validate mandatory fields + env, ok := s.Data["environment"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "environment") + } + container, ok := s.Data["container"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "container") + } + name, ok := s.Data["account_name"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "account_name") + } + key, ok := s.Data["account_key"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "account_key") + } + + return &storage.AzureStorageConfig{ + Env: string(env), + Container: string(container), + AccountName: string(name), + AccountKey: string(key), + }, nil +} + +func extractGCSConfigSecret(s *corev1.Secret) (*storage.GCSStorageConfig, error) { + // Extract and validate mandatory fields + bucket, ok := s.Data["bucketname"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "bucketname") + } + + return &storage.GCSStorageConfig{ + Bucket: string(bucket), + }, nil +} + +func extractS3ConfigSecret(s *corev1.Secret) (*storage.S3StorageConfig, error) { // Extract and validate mandatory fields endpoint, ok := s.Data["endpoint"] if !ok { @@ -29,12 +117,9 @@ func Extract(s *corev1.Secret) (*manifests.ObjectStorage, error) { } // Extract and validate optional fields - region, ok := s.Data["region"] - if !ok { - region = []byte("") - } + region := s.Data["region"] - return &manifests.ObjectStorage{ + return &storage.S3StorageConfig{ Endpoint: string(endpoint), Buckets: string(buckets), AccessKeyID: string(id), @@ -43,26 +128,66 @@ func Extract(s *corev1.Secret) (*manifests.ObjectStorage, error) { }, nil } -// ExtractGatewaySecret reads a k8s secret into a manifest tenant secret struct if valid. -func ExtractGatewaySecret(s *corev1.Secret, tenantName string) (*manifests.TenantSecrets, error) { +func extractSwiftConfigSecret(s *corev1.Secret) (*storage.SwiftStorageConfig, error) { // Extract and validate mandatory fields - clientID, ok := s.Data["clientID"] + url, ok := s.Data["auth_url"] if !ok { - return nil, kverrors.New("missing clientID field", "field", "clientID") + return nil, kverrors.New("missing secret field", "field", "auth_url") } - clientSecret, ok := s.Data["clientSecret"] + username, ok := s.Data["username"] if !ok { - return nil, kverrors.New("missing clientSecret field", "field", "clientSecret") + return nil, kverrors.New("missing secret field", "field", "username") } - issuerCAPath, ok := s.Data["issuerCAPath"] + userDomainName, ok := s.Data["user_domain_name"] if !ok { - return nil, kverrors.New("missing issuerCAPath field", "field", "issuerCAPath") + return nil, kverrors.New("missing secret field", "field", "user_domain_name") + } + userDomainID, ok := s.Data["user_domain_id"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "user_domain_id") + } + userID, ok := s.Data["user_id"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "user_id") + } + password, ok := s.Data["password"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "password") + } + domainID, ok := s.Data["domain_id"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "domain_id") + } + domainName, ok := s.Data["domain_name"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "domain_name") + } + containerName, ok := s.Data["container_name"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "container_name") } - return &manifests.TenantSecrets{ - TenantName: tenantName, - ClientID: string(clientID), - ClientSecret: string(clientSecret), - IssuerCAPath: string(issuerCAPath), + // Extract and validate optional fields + projectID := s.Data["project_id"] + projectName := s.Data["project_name"] + projectDomainID := s.Data["project_domain_id"] + projectDomainName := s.Data["project_domain_name"] + region := s.Data["region"] + + return &storage.SwiftStorageConfig{ + AuthURL: string(url), + Username: string(username), + UserDomainName: string(userDomainName), + UserDomainID: string(userDomainID), + UserID: string(userID), + Password: string(password), + DomainID: string(domainID), + DomainName: string(domainName), + ProjectID: string(projectID), + ProjectName: string(projectName), + ProjectDomainID: string(projectDomainID), + ProjectDomainName: string(projectDomainName), + Region: string(region), + Container: string(containerName), }, nil } diff --git a/operator/internal/handlers/internal/secrets/secrets_test.go b/operator/internal/handlers/internal/secrets/secrets_test.go index adaf76a43331b..62d57a27a598c 100644 --- a/operator/internal/handlers/internal/secrets/secrets_test.go +++ b/operator/internal/handlers/internal/secrets/secrets_test.go @@ -3,12 +3,120 @@ package secrets_test import ( "testing" + lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/handlers/internal/secrets" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" ) -func TestExtract(t *testing.T) { +func TestAzureExtract(t *testing.T) { + type test struct { + name string + secret *corev1.Secret + wantErr bool + } + table := []test{ + { + name: "missing environment", + secret: &corev1.Secret{}, + wantErr: true, + }, + { + name: "missing container", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("here"), + }, + }, + wantErr: true, + }, + { + name: "missing account_name", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + }, + }, + wantErr: true, + }, + { + name: "missing account_key", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + "account_name": []byte("id"), + }, + }, + wantErr: true, + }, + { + name: "all set", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + "account_name": []byte("id"), + "account_key": []byte("secret"), + }, + }, + }, + } + for _, tst := range table { + tst := tst + t.Run(tst.name, func(t *testing.T) { + t.Parallel() + + _, err := secrets.ExtractStorageSecret(tst.secret, lokiv1beta1.ObjectStorageSecretAzure) + if !tst.wantErr { + require.NoError(t, err) + } + if tst.wantErr { + require.NotNil(t, err) + } + }) + } +} + +func TestGCSExtract(t *testing.T) { + type test struct { + name string + secret *corev1.Secret + wantErr bool + } + table := []test{ + { + name: "missing bucketname", + secret: &corev1.Secret{}, + wantErr: true, + }, + { + name: "all set", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "bucketname": []byte("here"), + }, + }, + }, + } + for _, tst := range table { + tst := tst + t.Run(tst.name, func(t *testing.T) { + t.Parallel() + + _, err := secrets.ExtractStorageSecret(tst.secret, lokiv1beta1.ObjectStorageSecretGCS) + if !tst.wantErr { + require.NoError(t, err) + } + if tst.wantErr { + require.NotNil(t, err) + } + }) + } +} + +func TestS3Extract(t *testing.T) { type test struct { name string secret *corev1.Secret @@ -67,7 +175,152 @@ func TestExtract(t *testing.T) { t.Run(tst.name, func(t *testing.T) { t.Parallel() - _, err := secrets.Extract(tst.secret) + _, err := secrets.ExtractStorageSecret(tst.secret, lokiv1beta1.ObjectStorageSecretS3) + if !tst.wantErr { + require.NoError(t, err) + } + if tst.wantErr { + require.NotNil(t, err) + } + }) + } +} + +func TestSwiftExtract(t *testing.T) { + type test struct { + name string + secret *corev1.Secret + wantErr bool + } + table := []test{ + { + name: "missing auth_url", + secret: &corev1.Secret{}, + wantErr: true, + }, + { + name: "missing username", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + }, + }, + wantErr: true, + }, + { + name: "missing user_domain_name", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + }, + }, + wantErr: true, + }, + { + name: "missing user_domain_id", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + }, + }, + wantErr: true, + }, + { + name: "missing user_id", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + }, + }, + wantErr: true, + }, + { + name: "missing password", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + }, + }, + wantErr: true, + }, + { + name: "missing domain_id", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + "password": []byte("cred"), + }, + }, + wantErr: true, + }, + { + name: "missing domain_name", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + "password": []byte("cred"), + "domain_id": []byte("text"), + }, + }, + wantErr: true, + }, + { + name: "missing container_name", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + "password": []byte("cred"), + "domain_id": []byte("text"), + "domain_name": []byte("where"), + }, + }, + wantErr: true, + }, + { + name: "all set", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + "password": []byte("cred"), + "domain_id": []byte("text"), + "domain_name": []byte("where"), + "container_name": []byte("then"), + }, + }, + }, + } + for _, tst := range table { + tst := tst + t.Run(tst.name, func(t *testing.T) { + t.Parallel() + + _, err := secrets.ExtractStorageSecret(tst.secret, lokiv1beta1.ObjectStorageSecretSwift) if !tst.wantErr { require.NoError(t, err) } diff --git a/operator/internal/handlers/lokistack_create_or_update.go b/operator/internal/handlers/lokistack_create_or_update.go index c773de072ba00..cfe6230b7cf3f 100644 --- a/operator/internal/handlers/lokistack_create_or_update.go +++ b/operator/internal/handlers/lokistack_create_or_update.go @@ -49,19 +49,19 @@ func CreateOrUpdateLokiStack(ctx context.Context, req ctrl.Request, k k8s.Client gwImg = manifests.DefaultLokiStackGatewayImage } - var s3secret corev1.Secret + var storageSecret corev1.Secret key := client.ObjectKey{Name: stack.Spec.Storage.Secret.Name, Namespace: stack.Namespace} - if err := k.Get(ctx, key, &s3secret); err != nil { + if err := k.Get(ctx, key, &storageSecret); err != nil { if apierrors.IsNotFound(err) { return status.SetDegradedCondition(ctx, k, req, "Missing object storage secret", lokiv1beta1.ReasonMissingObjectStorageSecret, ) } - return kverrors.Wrap(err, "failed to lookup lokistack s3 secret", "name", key) + return kverrors.Wrap(err, "failed to lookup lokistack storage secret", "name", key) } - storage, err := secrets.Extract(&s3secret) + storage, err := secrets.ExtractStorageSecret(&storageSecret, stack.Spec.Storage.Secret.Type) if err != nil { return status.SetDegradedCondition(ctx, k, req, "Invalid object storage secret contents", diff --git a/operator/internal/handlers/lokistack_create_or_update_test.go b/operator/internal/handlers/lokistack_create_or_update_test.go index f2be3499ba134..522cbc55dd020 100644 --- a/operator/internal/handlers/lokistack_create_or_update_test.go +++ b/operator/internal/handlers/lokistack_create_or_update_test.go @@ -163,6 +163,7 @@ func TestCreateOrUpdateLokiStack_SetsNamespaceOnAllObjects(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -238,6 +239,7 @@ func TestCreateOrUpdateLokiStack_SetsOwnerRefOnAllObjects(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -338,6 +340,7 @@ func TestCreateOrUpdateLokiStack_WhenSetControllerRefInvalid_ContinueWithOtherOb Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -383,6 +386,7 @@ func TestCreateOrUpdateLokiStack_WhenGetReturnsNoError_UpdateObjects(t *testing. Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -479,6 +483,7 @@ func TestCreateOrUpdateLokiStack_WhenCreateReturnsError_ContinueWithOtherObjects Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -532,6 +537,7 @@ func TestCreateOrUpdateLokiStack_WhenUpdateReturnsError_ContinueWithOtherObjects Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -631,6 +637,7 @@ func TestCreateOrUpdateLokiStack_WhenMissingSecret_SetDegraded(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -682,6 +689,7 @@ func TestCreateOrUpdateLokiStack_WhenInvalidSecret_SetDegraded(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: invalidSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -741,6 +749,7 @@ func TestCreateOrUpdateLokiStack_WhenInvalidTenantsConfiguration_SetDegraded(t * Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -815,6 +824,7 @@ func TestCreateOrUpdateLokiStack_WhenMissingGatewaySecret_SetDegraded(t *testing Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -894,6 +904,7 @@ func TestCreateOrUpdateLokiStack_WhenInvalidGatewaySecret_SetDegraded(t *testing Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -977,6 +988,7 @@ func TestCreateOrUpdateLokiStack_MissingTenantsSpec_SetDegraded(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: nil, diff --git a/operator/internal/manifests/config.go b/operator/internal/manifests/config.go index 0dd803589fa02..0ac9ae169f5a3 100644 --- a/operator/internal/manifests/config.go +++ b/operator/internal/manifests/config.go @@ -63,13 +63,6 @@ func ConfigOptions(opt Options) config.Options { Port: grpcPort, }, StorageDirectory: dataDirectory, - ObjectStorage: config.ObjectStorage{ - Endpoint: opt.ObjectStorage.Endpoint, - Buckets: opt.ObjectStorage.Buckets, - Region: opt.ObjectStorage.Region, - AccessKeyID: opt.ObjectStorage.AccessKeyID, - AccessKeySecret: opt.ObjectStorage.AccessKeySecret, - }, QueryParallelism: config.Parallelism{ QuerierCPULimits: opt.ResourceRequirements.Querier.Requests.Cpu().Value(), QueryFrontendReplicas: opt.Stack.Template.QueryFrontend.Replicas, @@ -78,6 +71,7 @@ func ConfigOptions(opt Options) config.Options { Directory: walDirectory, IngesterMemoryRequest: opt.ResourceRequirements.Ingester.Requests.Memory().Value(), }, + ObjectStorage: opt.ObjectStorage, } } diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index ea2d66fdf0eff..832c4c7e89cc6 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -4,6 +4,7 @@ import ( "testing" lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" + "github.com/grafana/loki/operator/internal/manifests/storage" "github.com/stretchr/testify/require" ) @@ -16,9 +17,17 @@ chunk_store_config: enable_fifocache: true fifocache: max_size_bytes: 500MB +common: + storage: + s3: + s3: http://test.default.svc.cluster.local.:9000 + bucketnames: loki + region: us-east + access_key_id: test + secret_access_key: test123 + s3forcepathstyle: true compactor: compaction_interval: 2h - shared_store: s3 working_directory: /tmp/loki/compactor frontend: tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 @@ -146,13 +155,6 @@ storage_config: shared_store: s3 index_gateway_client: server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 - aws: - s3: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: test - secret_access_key: test123 - s3forcepathstyle: true tracing: enabled: false ` @@ -201,13 +203,6 @@ overrides: Port: 9095, }, StorageDirectory: "/tmp/loki", - ObjectStorage: ObjectStorage{ - Endpoint: "http://test.default.svc.cluster.local.:9000", - Region: "us-east", - Buckets: "loki", - AccessKeyID: "test", - AccessKeySecret: "test123", - }, QueryParallelism: Parallelism{ QuerierCPULimits: 2, QueryFrontendReplicas: 2, @@ -216,6 +211,15 @@ overrides: Directory: "/tmp/wal", IngesterMemoryRequest: 5000, }, + ObjectStorage: storage.Options{ + S3: &storage.S3StorageConfig{ + Endpoint: "http://test.default.svc.cluster.local.:9000", + Region: "us-east", + Buckets: "loki", + AccessKeyID: "test", + AccessKeySecret: "test123", + }, + }, } cfg, rCfg, err := Build(opts) require.NoError(t, err) @@ -232,9 +236,17 @@ chunk_store_config: enable_fifocache: true fifocache: max_size_bytes: 500MB +common: + storage: + s3: + s3: http://test.default.svc.cluster.local.:9000 + bucketnames: loki + region: us-east + access_key_id: test + secret_access_key: test123 + s3forcepathstyle: true compactor: compaction_interval: 2h - shared_store: s3 working_directory: /tmp/loki/compactor frontend: tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 @@ -362,13 +374,6 @@ storage_config: shared_store: s3 index_gateway_client: server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 - aws: - s3: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: test - secret_access_key: test123 - s3forcepathstyle: true tracing: enabled: false ` @@ -434,13 +439,6 @@ overrides: Port: 9095, }, StorageDirectory: "/tmp/loki", - ObjectStorage: ObjectStorage{ - Endpoint: "http://test.default.svc.cluster.local.:9000", - Region: "us-east", - Buckets: "loki", - AccessKeyID: "test", - AccessKeySecret: "test123", - }, QueryParallelism: Parallelism{ QuerierCPULimits: 2, QueryFrontendReplicas: 2, @@ -449,6 +447,15 @@ overrides: Directory: "/tmp/wal", IngesterMemoryRequest: 5000, }, + ObjectStorage: storage.Options{ + S3: &storage.S3StorageConfig{ + Endpoint: "http://test.default.svc.cluster.local.:9000", + Region: "us-east", + Buckets: "loki", + AccessKeyID: "test", + AccessKeySecret: "test123", + }, + }, } cfg, rCfg, err := Build(opts) require.NoError(t, err) @@ -495,13 +502,6 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { Port: 9095, }, StorageDirectory: "/tmp/loki", - ObjectStorage: ObjectStorage{ - Endpoint: "http://test.default.svc.cluster.local.:9000", - Region: "us-east", - Buckets: "loki", - AccessKeyID: "test", - AccessKeySecret: "test123", - }, QueryParallelism: Parallelism{ QuerierCPULimits: 2, QueryFrontendReplicas: 2, @@ -510,6 +510,15 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { Directory: "/tmp/wal", IngesterMemoryRequest: 5000, }, + ObjectStorage: storage.Options{ + S3: &storage.S3StorageConfig{ + Endpoint: "http://test.default.svc.cluster.local.:9000", + Region: "us-east", + Buckets: "loki", + AccessKeyID: "test", + AccessKeySecret: "test123", + }, + }, } cfg, rCfg, err := Build(opts) require.Error(t, err) diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index b611d3adcea45..22e009679cfdb 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -5,9 +5,47 @@ chunk_store_config: enable_fifocache: true fifocache: max_size_bytes: 500MB +common: + storage: + {{ with .ObjectStorage.Azure }} + azure: + environment: {{ .Env }} + container_name: {{ .Container }} + account_name: {{ .AccountName }} + account_key: {{ .AccountKey }} + {{ end }} + {{ with .ObjectStorage.GCS }} + gcs: + bucket_name: {{ .Bucket }} + {{ end }} + {{ with .ObjectStorage.S3 }} + s3: + s3: {{ .Endpoint }} + bucketnames: {{ .Buckets }} + region: {{ .Region }} + access_key_id: {{ .AccessKeyID }} + secret_access_key: {{ .AccessKeySecret }} + s3forcepathstyle: true + {{ end }} + {{ with .ObjectStorage.Swift }} + swift: + auth_url: {{ .AuthURL }} + username: {{ .Username }} + user_domain_name: {{ .UserDomainName }} + user_domain_id: {{ .UserDomainID }} + user_id: {{ .UserID }} + password: {{ .Password }} + domain_id: {{ .DomainID }} + domain_name: {{ .DomainName }} + project_id: {{ .ProjectID }} + project_name: {{ .ProjectName }} + project_domain_id: {{ .ProjectDomainID }} + project_domain_name: {{ .ProjectDomainName }} + region_name: {{ .Region }} + container_name: {{ .Container }} + {{ end }} compactor: compaction_interval: 2h - shared_store: s3 working_directory: {{ .StorageDirectory }}/compactor frontend: tail_proxy_url: http://{{ .Querier.FQDN }}:{{ .Querier.Port }} @@ -135,12 +173,5 @@ storage_config: shared_store: s3 index_gateway_client: server_address: dns:///{{ .IndexGateway.FQDN }}:{{ .IndexGateway.Port }} - aws: - s3: {{ .ObjectStorage.Endpoint }} - bucketnames: {{ .ObjectStorage.Buckets }} - region: {{ .ObjectStorage.Region }} - access_key_id: {{ .ObjectStorage.AccessKeyID }} - secret_access_key: {{ .ObjectStorage.AccessKeySecret }} - s3forcepathstyle: true tracing: enabled: false diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index 1280143d750aa..a0992cf52b97e 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -5,6 +5,7 @@ import ( "math" lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" + "github.com/grafana/loki/operator/internal/manifests/storage" ) // Options is used to render the loki-config.yaml file template @@ -18,9 +19,10 @@ type Options struct { Querier Address IndexGateway Address StorageDirectory string - ObjectStorage ObjectStorage QueryParallelism Parallelism WriteAheadLog WriteAheadLog + + ObjectStorage storage.Options } // Address FQDN and port for a k8s service. @@ -31,15 +33,6 @@ type Address struct { Port int } -// ObjectStorage for storage config. -type ObjectStorage struct { - Endpoint string - Region string - Buckets string - AccessKeyID string - AccessKeySecret string -} - // Parallelism for query processing parallelism // and rate limiting. type Parallelism struct { diff --git a/operator/internal/manifests/node_placement_test.go b/operator/internal/manifests/node_placement_test.go index 399ae9fc2a7c9..857524b2c0c6c 100644 --- a/operator/internal/manifests/node_placement_test.go +++ b/operator/internal/manifests/node_placement_test.go @@ -4,6 +4,7 @@ import ( "testing" lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" + "github.com/grafana/loki/operator/internal/manifests/storage" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" ) @@ -44,7 +45,7 @@ func TestTolerationsAreSetForEachComponent(t *testing.T) { }, }, }, - ObjectStorage: ObjectStorage{}, + ObjectStorage: storage.Options{}, } optsWithoutTolerations := Options{ @@ -70,7 +71,7 @@ func TestTolerationsAreSetForEachComponent(t *testing.T) { }, }, }, - ObjectStorage: ObjectStorage{}, + ObjectStorage: storage.Options{}, } t.Run("distributor", func(t *testing.T) { @@ -135,7 +136,7 @@ func TestNodeSelectorsAreSetForEachComponent(t *testing.T) { }, }, }, - ObjectStorage: ObjectStorage{}, + ObjectStorage: storage.Options{}, } optsWithoutNodeSelectors := Options{ @@ -161,7 +162,7 @@ func TestNodeSelectorsAreSetForEachComponent(t *testing.T) { }, }, }, - ObjectStorage: ObjectStorage{}, + ObjectStorage: storage.Options{}, } t.Run("distributor", func(t *testing.T) { diff --git a/operator/internal/manifests/options.go b/operator/internal/manifests/options.go index 0a0c9ba4c7c68..054b8e314285b 100644 --- a/operator/internal/manifests/options.go +++ b/operator/internal/manifests/options.go @@ -4,6 +4,7 @@ import ( lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/manifests/internal" "github.com/grafana/loki/operator/internal/manifests/openshift" + "github.com/grafana/loki/operator/internal/manifests/storage" ) // Options is a set of configuration values to use when building manifests such as resource sizes, etc. @@ -21,22 +22,13 @@ type Options struct { Stack lokiv1beta1.LokiStackSpec ResourceRequirements internal.ComponentResources - ObjectStorage ObjectStorage + ObjectStorage storage.Options OpenShiftOptions openshift.Options TenantSecrets []*TenantSecrets TenantConfigMap map[string]openshift.TenantData } -// ObjectStorage for storage config. -type ObjectStorage struct { - Endpoint string - Region string - Buckets string - AccessKeyID string - AccessKeySecret string -} - // FeatureFlags contains flags that activate various features type FeatureFlags struct { EnableCertificateSigningService bool diff --git a/operator/internal/manifests/storage/options.go b/operator/internal/manifests/storage/options.go new file mode 100644 index 0000000000000..254fe0a8d853e --- /dev/null +++ b/operator/internal/manifests/storage/options.go @@ -0,0 +1,50 @@ +package storage + +// Options is used to configure Loki to integrate with +// supported object storages. +type Options struct { + Azure *AzureStorageConfig + GCS *GCSStorageConfig + S3 *S3StorageConfig + Swift *SwiftStorageConfig +} + +// AzureStorageConfig for Azure storage config +type AzureStorageConfig struct { + Env string + Container string + AccountName string + AccountKey string +} + +// GCSStorageConfig for GCS storage config +type GCSStorageConfig struct { + Bucket string +} + +// S3StorageConfig for S3 storage config +type S3StorageConfig struct { + Endpoint string + Region string + Buckets string + AccessKeyID string + AccessKeySecret string +} + +// SwiftStorageConfig for Swift storage config +type SwiftStorageConfig struct { + AuthURL string + Username string + UserDomainName string + UserDomainID string + UserID string + Password string + DomainID string + DomainName string + ProjectID string + ProjectName string + ProjectDomainID string + ProjectDomainName string + Region string + Container string +}