From acd210c53c14643de5ad8fb24047b55272096e80 Mon Sep 17 00:00:00 2001 From: Gerard Vanloo Date: Fri, 18 Feb 2022 16:34:20 -0500 Subject: [PATCH 1/5] Adding storage secret type; Using common storage configuration --- operator/api/v1beta1/lokistack_types.go | 28 ++- operator/api/v1beta1/zz_generated.deepcopy.go | 1 + .../loki-operator.clusterserviceversion.yaml | 10 +- .../loki.grafana.com_lokistacks.yaml | 9 + .../bases/loki.grafana.com_lokistacks.yaml | 9 + .../loki-operator.clusterserviceversion.yaml | 10 +- operator/hack/lokistack_dev.yaml | 1 + operator/hack/lokistack_gateway_dev.yaml | 1 + operator/hack/lokistack_gateway_ocp.yaml | 1 + .../handlers/internal/secrets/secrets.go | 172 ++++++++++++++++-- .../handlers/lokistack_create_or_update.go | 2 +- operator/internal/manifests/config.go | 56 +++++- .../internal/config/loki-config.yaml | 44 ++++- .../manifests/internal/config/options.go | 58 ++++-- operator/internal/manifests/options.go | 39 ++++ 15 files changed, 394 insertions(+), 47 deletions(-) diff --git a/operator/api/v1beta1/lokistack_types.go b/operator/api/v1beta1/lokistack_types.go index 681017ec03856..d4b74dbeae83b 100644 --- a/operator/api/v1beta1/lokistack_types.go +++ b/operator/api/v1beta1/lokistack_types.go @@ -307,13 +307,39 @@ type LokiTemplateSpec struct { IndexGateway *LokiComponentSpec `json:"indexGateway,omitempty"` } +// ObjectStorageSecretType defines the type of storage which can be used with the Loki cluster. +// +// +kubebuilder:validation:Enum=Azure;GCS;Swift;S3 +type ObjectStorageSecretType string + +const ( + // ObjectStorageSecretAzure when using Azure for Loki storage + ObjectStorageSecretAzure ObjectStorageSecretType = "Azure" + + // ObjectStorageSecretGCS when using GCS for Loki storage + ObjectStorageSecretGCS ObjectStorageSecretType = "GCS" + + // ObjectStorageSecretS3 when using S3 for Loki storage + ObjectStorageSecretS3 ObjectStorageSecretType = "S3" + + // ObjectStorageSecretSwift when using Swift for Loki storage + ObjectStorageSecretSwift ObjectStorageSecretType = "Swift" +) + // ObjectStorageSecretSpec is a secret reference containing name only, no namespace. type ObjectStorageSecretSpec struct { + // Type of object storage that needs to be configured + // + // +required + // +kubebuilder:validation:Required + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:select:Azure","urn:alm:descriptor:com.tectonic.ui:select:GCS","urn:alm:descriptor:com.tectonic.ui:select:S3","urn:alm:descriptor:com.tectonic.ui:select:Swift"},displayName="Object Storage Secret Type" + Type ObjectStorageSecretType `json:"type"` + // Name of a secret in the namespace configured for object storage secrets. // // +required // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:io.kubernetes:Secret",displayName="Object Storage Secret" + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:io.kubernetes:Secret",displayName="Object Storage Secret Name" Name string `json:"name"` } diff --git a/operator/api/v1beta1/zz_generated.deepcopy.go b/operator/api/v1beta1/zz_generated.deepcopy.go index 81108bddc75f3..037f360de672a 100644 --- a/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/operator/api/v1beta1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. diff --git a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml index 97b67e2b66bb5..44cbcd95b70e7 100644 --- a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml @@ -240,10 +240,18 @@ spec: path: storage - description: Name of a secret in the namespace configured for object storage secrets. - displayName: Object Storage Secret + displayName: Object Storage Secret Name path: storage.secret.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret + - description: Type of object storage that needs to be configured + displayName: Object Storage Secret Type + path: storage.secret.type + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:Azure + - urn:alm:descriptor:com.tectonic.ui:select:GCS + - urn:alm:descriptor:com.tectonic.ui:select:S3 + - urn:alm:descriptor:com.tectonic.ui:select:Swift - description: Storage class name defines the storage class for ingester/querier PVCs. displayName: Storage Class Name diff --git a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml index 1bb04ccee8e84..1ed0d8ff24765 100644 --- a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml @@ -220,8 +220,17 @@ spec: description: Name of a secret in the namespace configured for object storage secrets. type: string + type: + description: Type of object storage that needs to be configured + enum: + - Azure + - GCS + - Swift + - S3 + type: string required: - name + - type type: object required: - secret diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index fd67db9befb2c..77b9aa2625bd8 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -173,8 +173,17 @@ spec: name: description: Name of a secret in the namespace configured for object storage secrets. type: string + type: + description: Type of object storage that needs to be configured + enum: + - Azure + - GCS + - Swift + - S3 + type: string required: - name + - type type: object required: - secret diff --git a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml index 9c6a505b81e36..b3cfd76996524 100644 --- a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml @@ -219,10 +219,18 @@ spec: path: storage - description: Name of a secret in the namespace configured for object storage secrets. - displayName: Object Storage Secret + displayName: Object Storage Secret Name path: storage.secret.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret + - description: Type of object storage that needs to be configured + displayName: Object Storage Secret Type + path: storage.secret.type + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:Azure + - urn:alm:descriptor:com.tectonic.ui:select:GCS + - urn:alm:descriptor:com.tectonic.ui:select:S3 + - urn:alm:descriptor:com.tectonic.ui:select:Swift - description: Storage class name defines the storage class for ingester/querier PVCs. displayName: Storage Class Name diff --git a/operator/hack/lokistack_dev.yaml b/operator/hack/lokistack_dev.yaml index 2649ebe9ffa79..b640a4adb2243 100644 --- a/operator/hack/lokistack_dev.yaml +++ b/operator/hack/lokistack_dev.yaml @@ -8,4 +8,5 @@ spec: storage: secret: name: test + type: S3 storageClassName: standard diff --git a/operator/hack/lokistack_gateway_dev.yaml b/operator/hack/lokistack_gateway_dev.yaml index aee3cafc6ba53..1202cecc05cb4 100644 --- a/operator/hack/lokistack_gateway_dev.yaml +++ b/operator/hack/lokistack_gateway_dev.yaml @@ -8,6 +8,7 @@ spec: storage: secret: name: test + type: S3 storageClassName: gp2 tenants: mode: static diff --git a/operator/hack/lokistack_gateway_ocp.yaml b/operator/hack/lokistack_gateway_ocp.yaml index 8297be48e638c..c15e643385995 100644 --- a/operator/hack/lokistack_gateway_ocp.yaml +++ b/operator/hack/lokistack_gateway_ocp.yaml @@ -8,6 +8,7 @@ spec: storage: secret: name: test + type: S3 storageClassName: gp2 tenants: mode: openshift-logging diff --git a/operator/internal/handlers/internal/secrets/secrets.go b/operator/internal/handlers/internal/secrets/secrets.go index 88f2f2a4a39cb..0847cae4a4946 100644 --- a/operator/internal/handlers/internal/secrets/secrets.go +++ b/operator/internal/handlers/internal/secrets/secrets.go @@ -2,13 +2,100 @@ package secrets import ( "github.com/ViaQ/logerr/kverrors" + lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/manifests" corev1 "k8s.io/api/core/v1" ) // Extract reads a k8s secret into a manifest object storage struct if valid. -func Extract(s *corev1.Secret) (*manifests.ObjectStorage, error) { +func Extract(s *corev1.Secret, t lokiv1beta1.ObjectStorageSecretType) (*manifests.ObjectStorage, error) { + var err error + storage := manifests.ObjectStorage{} + + switch t { + case lokiv1beta1.ObjectStorageSecretAzure: + storage.Azure, err = extractAzureConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretGCS: + storage.GCS, err = extractGCSConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretS3: + storage.S3, err = extractS3ConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretSwift: + storage.Swift, err = extractSwiftConfigSecret(s) + default: + return nil, kverrors.New("unknown secret type", "type", t) + } + + if err != nil { + return nil, err + } + return &storage, nil +} + +// ExtractGatewaySecret reads a k8s secret into a manifest tenant secret struct if valid. +func ExtractGatewaySecret(s *corev1.Secret, tenantName string) (*manifests.TenantSecrets, error) { + // Extract and validate mandatory fields + clientID, ok := s.Data["clientID"] + if !ok { + return nil, kverrors.New("missing clientID field", "field", "clientID") + } + clientSecret, ok := s.Data["clientSecret"] + if !ok { + return nil, kverrors.New("missing clientSecret field", "field", "clientSecret") + } + issuerCAPath, ok := s.Data["issuerCAPath"] + if !ok { + return nil, kverrors.New("missing issuerCAPath field", "field", "issuerCAPath") + } + + return &manifests.TenantSecrets{ + TenantName: tenantName, + ClientID: string(clientID), + ClientSecret: string(clientSecret), + IssuerCAPath: string(issuerCAPath), + }, nil +} + +func extractAzureConfigSecret(s *corev1.Secret) (*manifests.AzureConfig, error) { + // Extract and validate mandatory fields + env, ok := s.Data["environment"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "environment") + } + container, ok := s.Data["container"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "container") + } + name, ok := s.Data["account_name"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "account_name") + } + key, ok := s.Data["account_key"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "account_key") + } + + return &manifests.AzureConfig{ + Env: string(env), + Container: string(container), + AccountName: string(name), + AccountKey: string(key), + }, nil +} + +func extractGCSConfigSecret(s *corev1.Secret) (*manifests.GCSConfig, error) { + // Extract and validate mandatory fields + bucket, ok := s.Data["bucketname"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "bucketname") + } + + return &manifests.GCSConfig{ + Bucket: string(bucket), + }, nil +} + +func extractS3ConfigSecret(s *corev1.Secret) (*manifests.S3Config, error) { // Extract and validate mandatory fields endpoint, ok := s.Data["endpoint"] if !ok { @@ -34,7 +121,7 @@ func Extract(s *corev1.Secret) (*manifests.ObjectStorage, error) { region = []byte("") } - return &manifests.ObjectStorage{ + return &manifests.S3Config{ Endpoint: string(endpoint), Buckets: string(buckets), AccessKeyID: string(id), @@ -43,26 +130,81 @@ func Extract(s *corev1.Secret) (*manifests.ObjectStorage, error) { }, nil } -// ExtractGatewaySecret reads a k8s secret into a manifest tenant secret struct if valid. -func ExtractGatewaySecret(s *corev1.Secret, tenantName string) (*manifests.TenantSecrets, error) { +func extractSwiftConfigSecret(s *corev1.Secret) (*manifests.SwiftConfig, error) { // Extract and validate mandatory fields - clientID, ok := s.Data["clientID"] + url, ok := s.Data["auth_url"] if !ok { - return nil, kverrors.New("missing clientID field", "field", "clientID") + return nil, kverrors.New("missing secret field", "field", "auth_url") } - clientSecret, ok := s.Data["clientSecret"] + username, ok := s.Data["username"] if !ok { - return nil, kverrors.New("missing clientSecret field", "field", "clientSecret") + return nil, kverrors.New("missing secret field", "field", "username") } - issuerCAPath, ok := s.Data["issuerCAPath"] + userDomainName, ok := s.Data["user_domain_name"] if !ok { - return nil, kverrors.New("missing issuerCAPath field", "field", "issuerCAPath") + return nil, kverrors.New("missing secret field", "field", "user_domain_name") + } + userDomainID, ok := s.Data["user_domain_id"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "user_domain_id") + } + userID, ok := s.Data["user_id"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "user_id") + } + password, ok := s.Data["password"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "password") + } + domainID, ok := s.Data["domain_id"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "domain_id") + } + domainName, ok := s.Data["domain_name"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "domain_name") + } + containerName, ok := s.Data["container_name"] + if !ok { + return nil, kverrors.New("missing secret field", "field", "container_name") } - return &manifests.TenantSecrets{ - TenantName: tenantName, - ClientID: string(clientID), - ClientSecret: string(clientSecret), - IssuerCAPath: string(issuerCAPath), + // Extract and validate optional fields + projectID, ok := s.Data["project_id"] + if !ok { + projectID = []byte("") + } + projectName, ok := s.Data["project_name"] + if !ok { + projectName = []byte("") + } + projectDomainID, ok := s.Data["project_domain_id"] + if !ok { + projectDomainID = []byte("") + } + projectDomainName, ok := s.Data["project_domain_name"] + if !ok { + projectDomainName = []byte("") + } + region, ok := s.Data["region"] + if !ok { + region = []byte("") + } + + return &manifests.SwiftConfig{ + AuthURL: string(url), + Username: string(username), + UserDomainName: string(userDomainName), + UserDomainID: string(userDomainID), + UserID: string(userID), + Password: string(password), + DomainID: string(domainID), + DomainName: string(domainName), + ProjectID: string(projectID), + ProjectName: string(projectName), + ProjectDomainID: string(projectDomainID), + ProjectDomainName: string(projectDomainName), + Region: string(region), + Container: string(containerName), }, nil } diff --git a/operator/internal/handlers/lokistack_create_or_update.go b/operator/internal/handlers/lokistack_create_or_update.go index c773de072ba00..870d8aff68fcc 100644 --- a/operator/internal/handlers/lokistack_create_or_update.go +++ b/operator/internal/handlers/lokistack_create_or_update.go @@ -61,7 +61,7 @@ func CreateOrUpdateLokiStack(ctx context.Context, req ctrl.Request, k k8s.Client return kverrors.Wrap(err, "failed to lookup lokistack s3 secret", "name", key) } - storage, err := secrets.Extract(&s3secret) + storage, err := secrets.Extract(&s3secret, stack.Spec.Storage.Secret.Type) if err != nil { return status.SetDegradedCondition(ctx, k, req, "Invalid object storage secret contents", diff --git a/operator/internal/manifests/config.go b/operator/internal/manifests/config.go index 0dd803589fa02..04e547e28010b 100644 --- a/operator/internal/manifests/config.go +++ b/operator/internal/manifests/config.go @@ -42,6 +42,49 @@ func LokiConfigMap(opt Options) (*corev1.ConfigMap, string, error) { // ConfigOptions converts Options to config.Options func ConfigOptions(opt Options) config.Options { + var azureStorage *config.AzureObjectStorage + var gcsStorage *config.GCSObjectStorage + var s3Storage *config.S3ObjectStorage + var swiftStorage *config.SwiftObjectStorage + + if opt.ObjectStorage.Azure != nil { + azureStorage = &config.AzureObjectStorage{ + Env: opt.ObjectStorage.Azure.Env, + Container: opt.ObjectStorage.Azure.Container, + AccountName: opt.ObjectStorage.Azure.AccountName, + AccountKey: opt.ObjectStorage.Azure.AccountKey, + } + } else if opt.ObjectStorage.GCS != nil { + gcsStorage = &config.GCSObjectStorage{ + Bucket: opt.ObjectStorage.GCS.Bucket, + } + } else if opt.ObjectStorage.S3 != nil { + s3Storage = &config.S3ObjectStorage{ + Endpoint: opt.ObjectStorage.S3.Endpoint, + Region: opt.ObjectStorage.S3.Region, + Buckets: opt.ObjectStorage.S3.Buckets, + AccessKeyID: opt.ObjectStorage.S3.AccessKeyID, + AccessKeySecret: opt.ObjectStorage.S3.AccessKeySecret, + } + } else if opt.ObjectStorage.Swift != nil { + swiftStorage = &config.SwiftObjectStorage{ + AuthURL: opt.ObjectStorage.Swift.AuthURL, + Username: opt.ObjectStorage.Swift.Username, + UserDomainName: opt.ObjectStorage.Swift.UserDomainName, + UserDomainID: opt.ObjectStorage.Swift.UserDomainID, + UserID: opt.ObjectStorage.Swift.UserID, + Password: opt.ObjectStorage.Swift.Password, + DomainID: opt.ObjectStorage.Swift.DomainID, + DomainName: opt.ObjectStorage.Swift.DomainName, + ProjectID: opt.ObjectStorage.Swift.ProjectID, + ProjectName: opt.ObjectStorage.Swift.ProjectName, + ProjectDomainID: opt.ObjectStorage.Swift.ProjectDomainID, + ProjectDomainName: opt.ObjectStorage.Swift.ProjectDomainName, + Region: opt.ObjectStorage.Swift.Region, + Container: opt.ObjectStorage.Swift.Container, + } + } + return config.Options{ Stack: opt.Stack, Namespace: opt.Namespace, @@ -62,14 +105,11 @@ func ConfigOptions(opt Options) config.Options { FQDN: fqdn(NewIndexGatewayGRPCService(opt).GetName(), opt.Namespace), Port: grpcPort, }, - StorageDirectory: dataDirectory, - ObjectStorage: config.ObjectStorage{ - Endpoint: opt.ObjectStorage.Endpoint, - Buckets: opt.ObjectStorage.Buckets, - Region: opt.ObjectStorage.Region, - AccessKeyID: opt.ObjectStorage.AccessKeyID, - AccessKeySecret: opt.ObjectStorage.AccessKeySecret, - }, + StorageDirectory: dataDirectory, + AzureObjectStorage: azureStorage, + GCSObjectStorage: gcsStorage, + S3ObjectStorage: s3Storage, + SwiftObjectStorage: swiftStorage, QueryParallelism: config.Parallelism{ QuerierCPULimits: opt.ResourceRequirements.Querier.Requests.Cpu().Value(), QueryFrontendReplicas: opt.Stack.Template.QueryFrontend.Replicas, diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index b611d3adcea45..36b23ba1bd5d4 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -5,9 +5,44 @@ chunk_store_config: enable_fifocache: true fifocache: max_size_bytes: 500MB +common: + storage: + {{ if .AzureObjectStorage }} + azure: + environment: {{ .AzureObjectStorage.Env }} + container_name: {{ .AzureObjectStorage.Container }} + account_name: {{ .AzureObjectStorage.AccountName }} + account_key: {{ .AzureObjectStorage.AccountKey }} + {{ else if .GCSObjectStorage }} + gcs: + bucket_name: {{ .GCSObjectStorage.Bucket }} + {{ else if .S3ObjectStorage }} + s3: + s3: {{ .S3ObjectStorage.Endpoint }} + bucketnames: {{ .S3ObjectStorage.Buckets }} + region: {{ .S3ObjectStorage.Region }} + access_key_id: {{ .S3ObjectStorage.AccessKeyID }} + secret_access_key: {{ .S3ObjectStorage.AccessKeySecret }} + s3forcepathstyle: true + {{ else if .SwiftObjectStorage }} + swift: + auth_url: {{ .SwiftObjectStorage.AuthURL }} + username: {{ .SwiftObjectStorage.Username }} + user_domain_name: {{ .SwiftObjectStorage.UserDomainName }} + user_domain_id: {{ .SwiftObjectStorage.UserDomainID }} + user_id: {{ .SwiftObjectStorage.UserID }} + password: {{ .SwiftObjectStorage.Password }} + domain_id: {{ .SwiftObjectStorage.DomainID }} + domain_name: {{ .SwiftObjectStorage.DomainName }} + project_id: {{ .SwiftObjectStorage.ProjectID }} + project_name: {{ .SwiftObjectStorage.ProjectName }} + project_domain_id: {{ .SwiftObjectStorage.ProjectDomainID }} + project_domain_name: {{ .SwiftObjectStorage.ProjectDomainName }} + region_name: {{ .SwiftObjectStorage.Region }} + container_name: {{ .SwiftObjectStorage.Container }} + {{ end }} compactor: compaction_interval: 2h - shared_store: s3 working_directory: {{ .StorageDirectory }}/compactor frontend: tail_proxy_url: http://{{ .Querier.FQDN }}:{{ .Querier.Port }} @@ -135,12 +170,5 @@ storage_config: shared_store: s3 index_gateway_client: server_address: dns:///{{ .IndexGateway.FQDN }}:{{ .IndexGateway.Port }} - aws: - s3: {{ .ObjectStorage.Endpoint }} - bucketnames: {{ .ObjectStorage.Buckets }} - region: {{ .ObjectStorage.Region }} - access_key_id: {{ .ObjectStorage.AccessKeyID }} - secret_access_key: {{ .ObjectStorage.AccessKeySecret }} - s3forcepathstyle: true tracing: enabled: false diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index 1280143d750aa..7a90f9c96d751 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -11,16 +11,19 @@ import ( type Options struct { Stack lokiv1beta1.LokiStackSpec - Namespace string - Name string - FrontendWorker Address - GossipRing Address - Querier Address - IndexGateway Address - StorageDirectory string - ObjectStorage ObjectStorage - QueryParallelism Parallelism - WriteAheadLog WriteAheadLog + Namespace string + Name string + FrontendWorker Address + GossipRing Address + Querier Address + IndexGateway Address + StorageDirectory string + AzureObjectStorage *AzureObjectStorage + GCSObjectStorage *GCSObjectStorage + S3ObjectStorage *S3ObjectStorage + SwiftObjectStorage *SwiftObjectStorage + QueryParallelism Parallelism + WriteAheadLog WriteAheadLog } // Address FQDN and port for a k8s service. @@ -31,8 +34,21 @@ type Address struct { Port int } -// ObjectStorage for storage config. -type ObjectStorage struct { +// AzureObjectStorage for Azure storage config +type AzureObjectStorage struct { + Env string + Container string + AccountName string + AccountKey string +} + +// GCSObjectStorage for GCS storage config +type GCSObjectStorage struct { + Bucket string +} + +// S3ObjectStorage for S3 storage config +type S3ObjectStorage struct { Endpoint string Region string Buckets string @@ -40,6 +56,24 @@ type ObjectStorage struct { AccessKeySecret string } +// SwiftObjectStorage for Swift storage config +type SwiftObjectStorage struct { + AuthURL string + Username string + UserDomainName string + UserDomainID string + UserID string + Password string + DomainID string + DomainName string + ProjectID string + ProjectName string + ProjectDomainID string + ProjectDomainName string + Region string + Container string +} + // Parallelism for query processing parallelism // and rate limiting. type Parallelism struct { diff --git a/operator/internal/manifests/options.go b/operator/internal/manifests/options.go index 0a0c9ba4c7c68..199b9c7362921 100644 --- a/operator/internal/manifests/options.go +++ b/operator/internal/manifests/options.go @@ -30,6 +30,27 @@ type Options struct { // ObjectStorage for storage config. type ObjectStorage struct { + Azure *AzureConfig + GCS *GCSConfig + S3 *S3Config + Swift *SwiftConfig +} + +// AzureConfig for Azure storage config +type AzureConfig struct { + Env string + Container string + AccountName string + AccountKey string +} + +// GCSConfig for GCS storage config +type GCSConfig struct { + Bucket string +} + +// S3Config for S3 storage config +type S3Config struct { Endpoint string Region string Buckets string @@ -37,6 +58,24 @@ type ObjectStorage struct { AccessKeySecret string } +// SwiftConfig for Swift storage config +type SwiftConfig struct { + AuthURL string + Username string + UserDomainName string + UserDomainID string + UserID string + Password string + DomainID string + DomainName string + ProjectID string + ProjectName string + ProjectDomainID string + ProjectDomainName string + Region string + Container string +} + // FeatureFlags contains flags that activate various features type FeatureFlags struct { EnableCertificateSigningService bool From 6050180b930ebf5fb70731a8a0c530fedcd4ff13 Mon Sep 17 00:00:00 2001 From: Gerard Vanloo Date: Fri, 18 Feb 2022 16:34:43 -0500 Subject: [PATCH 2/5] Updating test cases --- operator/cmd/loki-broker/main.go | 22 +- .../handlers/internal/secrets/secrets.go | 8 +- .../handlers/internal/secrets/secrets_test.go | 257 +++++++++++++++++- .../handlers/lokistack_create_or_update.go | 2 +- .../lokistack_create_or_update_test.go | 12 + .../manifests/internal/config/build_test.go | 40 +-- 6 files changed, 305 insertions(+), 36 deletions(-) diff --git a/operator/cmd/loki-broker/main.go b/operator/cmd/loki-broker/main.go index 979193a0bb311..ef6df099e78f4 100644 --- a/operator/cmd/loki-broker/main.go +++ b/operator/cmd/loki-broker/main.go @@ -39,12 +39,14 @@ func (c *config) registerFlags(f *flag.FlagSet) { f.BoolVar(&c.featureFlags.EnableTLSServiceMonitorConfig, "with-tls-service-monitors", false, "Enable TLS endpoint for service monitors.") f.BoolVar(&c.featureFlags.EnableGateway, "with-lokistack-gateway", false, "Enables the manifest creation for the entire lokistack-gateway.") // Object storage options - c.objectStorage = manifests.ObjectStorage{} - f.StringVar(&c.objectStorage.Endpoint, "object-storage.endpoint", "", "The S3 endpoint location.") - f.StringVar(&c.objectStorage.Buckets, "object-storage.buckets", "", "A comma-separated list of S3 buckets.") - f.StringVar(&c.objectStorage.Region, "object-storage.region", "", "An S3 region.") - f.StringVar(&c.objectStorage.AccessKeyID, "object-storage.access-key-id", "", "The access key id for S3.") - f.StringVar(&c.objectStorage.AccessKeySecret, "object-storage.access-key-secret", "", "The access key secret for S3.") + c.objectStorage = manifests.ObjectStorage{ + S3: &manifests.S3Config{}, + } + f.StringVar(&c.objectStorage.S3.Endpoint, "object-storage.endpoint", "", "The S3 endpoint location.") + f.StringVar(&c.objectStorage.S3.Buckets, "object-storage.buckets", "", "A comma-separated list of S3 buckets.") + f.StringVar(&c.objectStorage.S3.Region, "object-storage.region", "", "An S3 region.") + f.StringVar(&c.objectStorage.S3.AccessKeyID, "object-storage.access-key-id", "", "The access key id for S3.") + f.StringVar(&c.objectStorage.S3.AccessKeySecret, "object-storage.access-key-secret", "", "The access key secret for S3.") // Input and output file/dir options f.StringVar(&c.crFilepath, "custom-resource.path", "", "Path to a custom resource YAML file.") f.StringVar(&c.writeToDir, "output.write-dir", "", "write each file to the specified directory.") @@ -64,19 +66,19 @@ func (c *config) validateFlags() { os.Exit(1) } // Validate manifests.objectStorage - if cfg.objectStorage.Endpoint == "" { + if cfg.objectStorage.S3.Endpoint == "" { log.Info("-object.storage.endpoint flag is required") os.Exit(1) } - if cfg.objectStorage.Buckets == "" { + if cfg.objectStorage.S3.Buckets == "" { log.Info("-object.storage.buckets flag is required") os.Exit(1) } - if cfg.objectStorage.AccessKeyID == "" { + if cfg.objectStorage.S3.AccessKeyID == "" { log.Info("-object.storage.access.key.id flag is required") os.Exit(1) } - if cfg.objectStorage.AccessKeySecret == "" { + if cfg.objectStorage.S3.AccessKeySecret == "" { log.Info("-object.storage.access.key.secret flag is required") os.Exit(1) } diff --git a/operator/internal/handlers/internal/secrets/secrets.go b/operator/internal/handlers/internal/secrets/secrets.go index 0847cae4a4946..a551e82eada92 100644 --- a/operator/internal/handlers/internal/secrets/secrets.go +++ b/operator/internal/handlers/internal/secrets/secrets.go @@ -8,12 +8,12 @@ import ( corev1 "k8s.io/api/core/v1" ) -// Extract reads a k8s secret into a manifest object storage struct if valid. -func Extract(s *corev1.Secret, t lokiv1beta1.ObjectStorageSecretType) (*manifests.ObjectStorage, error) { +// ExtractStorageSecret reads a k8s secret into a manifest object storage struct if valid. +func ExtractStorageSecret(s *corev1.Secret, secretType lokiv1beta1.ObjectStorageSecretType) (*manifests.ObjectStorage, error) { var err error storage := manifests.ObjectStorage{} - switch t { + switch secretType { case lokiv1beta1.ObjectStorageSecretAzure: storage.Azure, err = extractAzureConfigSecret(s) case lokiv1beta1.ObjectStorageSecretGCS: @@ -23,7 +23,7 @@ func Extract(s *corev1.Secret, t lokiv1beta1.ObjectStorageSecretType) (*manifest case lokiv1beta1.ObjectStorageSecretSwift: storage.Swift, err = extractSwiftConfigSecret(s) default: - return nil, kverrors.New("unknown secret type", "type", t) + return nil, kverrors.New("unknown secret type", "type", secretType) } if err != nil { diff --git a/operator/internal/handlers/internal/secrets/secrets_test.go b/operator/internal/handlers/internal/secrets/secrets_test.go index adaf76a43331b..62d57a27a598c 100644 --- a/operator/internal/handlers/internal/secrets/secrets_test.go +++ b/operator/internal/handlers/internal/secrets/secrets_test.go @@ -3,12 +3,120 @@ package secrets_test import ( "testing" + lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/handlers/internal/secrets" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" ) -func TestExtract(t *testing.T) { +func TestAzureExtract(t *testing.T) { + type test struct { + name string + secret *corev1.Secret + wantErr bool + } + table := []test{ + { + name: "missing environment", + secret: &corev1.Secret{}, + wantErr: true, + }, + { + name: "missing container", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("here"), + }, + }, + wantErr: true, + }, + { + name: "missing account_name", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + }, + }, + wantErr: true, + }, + { + name: "missing account_key", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + "account_name": []byte("id"), + }, + }, + wantErr: true, + }, + { + name: "all set", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "environment": []byte("here"), + "container": []byte("this,that"), + "account_name": []byte("id"), + "account_key": []byte("secret"), + }, + }, + }, + } + for _, tst := range table { + tst := tst + t.Run(tst.name, func(t *testing.T) { + t.Parallel() + + _, err := secrets.ExtractStorageSecret(tst.secret, lokiv1beta1.ObjectStorageSecretAzure) + if !tst.wantErr { + require.NoError(t, err) + } + if tst.wantErr { + require.NotNil(t, err) + } + }) + } +} + +func TestGCSExtract(t *testing.T) { + type test struct { + name string + secret *corev1.Secret + wantErr bool + } + table := []test{ + { + name: "missing bucketname", + secret: &corev1.Secret{}, + wantErr: true, + }, + { + name: "all set", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "bucketname": []byte("here"), + }, + }, + }, + } + for _, tst := range table { + tst := tst + t.Run(tst.name, func(t *testing.T) { + t.Parallel() + + _, err := secrets.ExtractStorageSecret(tst.secret, lokiv1beta1.ObjectStorageSecretGCS) + if !tst.wantErr { + require.NoError(t, err) + } + if tst.wantErr { + require.NotNil(t, err) + } + }) + } +} + +func TestS3Extract(t *testing.T) { type test struct { name string secret *corev1.Secret @@ -67,7 +175,152 @@ func TestExtract(t *testing.T) { t.Run(tst.name, func(t *testing.T) { t.Parallel() - _, err := secrets.Extract(tst.secret) + _, err := secrets.ExtractStorageSecret(tst.secret, lokiv1beta1.ObjectStorageSecretS3) + if !tst.wantErr { + require.NoError(t, err) + } + if tst.wantErr { + require.NotNil(t, err) + } + }) + } +} + +func TestSwiftExtract(t *testing.T) { + type test struct { + name string + secret *corev1.Secret + wantErr bool + } + table := []test{ + { + name: "missing auth_url", + secret: &corev1.Secret{}, + wantErr: true, + }, + { + name: "missing username", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + }, + }, + wantErr: true, + }, + { + name: "missing user_domain_name", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + }, + }, + wantErr: true, + }, + { + name: "missing user_domain_id", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + }, + }, + wantErr: true, + }, + { + name: "missing user_id", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + }, + }, + wantErr: true, + }, + { + name: "missing password", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + }, + }, + wantErr: true, + }, + { + name: "missing domain_id", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + "password": []byte("cred"), + }, + }, + wantErr: true, + }, + { + name: "missing domain_name", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + "password": []byte("cred"), + "domain_id": []byte("text"), + }, + }, + wantErr: true, + }, + { + name: "missing container_name", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + "password": []byte("cred"), + "domain_id": []byte("text"), + "domain_name": []byte("where"), + }, + }, + wantErr: true, + }, + { + name: "all set", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "auth_url": []byte("here"), + "username": []byte("this,that"), + "user_domain_name": []byte("id"), + "user_domain_id": []byte("secret"), + "user_id": []byte("there"), + "password": []byte("cred"), + "domain_id": []byte("text"), + "domain_name": []byte("where"), + "container_name": []byte("then"), + }, + }, + }, + } + for _, tst := range table { + tst := tst + t.Run(tst.name, func(t *testing.T) { + t.Parallel() + + _, err := secrets.ExtractStorageSecret(tst.secret, lokiv1beta1.ObjectStorageSecretSwift) if !tst.wantErr { require.NoError(t, err) } diff --git a/operator/internal/handlers/lokistack_create_or_update.go b/operator/internal/handlers/lokistack_create_or_update.go index 870d8aff68fcc..3fb781a996270 100644 --- a/operator/internal/handlers/lokistack_create_or_update.go +++ b/operator/internal/handlers/lokistack_create_or_update.go @@ -61,7 +61,7 @@ func CreateOrUpdateLokiStack(ctx context.Context, req ctrl.Request, k k8s.Client return kverrors.Wrap(err, "failed to lookup lokistack s3 secret", "name", key) } - storage, err := secrets.Extract(&s3secret, stack.Spec.Storage.Secret.Type) + storage, err := secrets.ExtractStorageSecret(&s3secret, stack.Spec.Storage.Secret.Type) if err != nil { return status.SetDegradedCondition(ctx, k, req, "Invalid object storage secret contents", diff --git a/operator/internal/handlers/lokistack_create_or_update_test.go b/operator/internal/handlers/lokistack_create_or_update_test.go index f2be3499ba134..522cbc55dd020 100644 --- a/operator/internal/handlers/lokistack_create_or_update_test.go +++ b/operator/internal/handlers/lokistack_create_or_update_test.go @@ -163,6 +163,7 @@ func TestCreateOrUpdateLokiStack_SetsNamespaceOnAllObjects(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -238,6 +239,7 @@ func TestCreateOrUpdateLokiStack_SetsOwnerRefOnAllObjects(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -338,6 +340,7 @@ func TestCreateOrUpdateLokiStack_WhenSetControllerRefInvalid_ContinueWithOtherOb Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -383,6 +386,7 @@ func TestCreateOrUpdateLokiStack_WhenGetReturnsNoError_UpdateObjects(t *testing. Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -479,6 +483,7 @@ func TestCreateOrUpdateLokiStack_WhenCreateReturnsError_ContinueWithOtherObjects Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -532,6 +537,7 @@ func TestCreateOrUpdateLokiStack_WhenUpdateReturnsError_ContinueWithOtherObjects Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -631,6 +637,7 @@ func TestCreateOrUpdateLokiStack_WhenMissingSecret_SetDegraded(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -682,6 +689,7 @@ func TestCreateOrUpdateLokiStack_WhenInvalidSecret_SetDegraded(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: invalidSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, }, @@ -741,6 +749,7 @@ func TestCreateOrUpdateLokiStack_WhenInvalidTenantsConfiguration_SetDegraded(t * Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -815,6 +824,7 @@ func TestCreateOrUpdateLokiStack_WhenMissingGatewaySecret_SetDegraded(t *testing Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -894,6 +904,7 @@ func TestCreateOrUpdateLokiStack_WhenInvalidGatewaySecret_SetDegraded(t *testing Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: &lokiv1beta1.TenantsSpec{ @@ -977,6 +988,7 @@ func TestCreateOrUpdateLokiStack_MissingTenantsSpec_SetDegraded(t *testing.T) { Storage: lokiv1beta1.ObjectStorageSpec{ Secret: lokiv1beta1.ObjectStorageSecretSpec{ Name: defaultSecret.Name, + Type: lokiv1beta1.ObjectStorageSecretS3, }, }, Tenants: nil, diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index ea2d66fdf0eff..478cee509f1ce 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -16,9 +16,17 @@ chunk_store_config: enable_fifocache: true fifocache: max_size_bytes: 500MB +common: + storage: + s3: + s3: http://test.default.svc.cluster.local.:9000 + bucketnames: loki + region: us-east + access_key_id: test + secret_access_key: test123 + s3forcepathstyle: true compactor: compaction_interval: 2h - shared_store: s3 working_directory: /tmp/loki/compactor frontend: tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 @@ -146,13 +154,6 @@ storage_config: shared_store: s3 index_gateway_client: server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 - aws: - s3: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: test - secret_access_key: test123 - s3forcepathstyle: true tracing: enabled: false ` @@ -201,7 +202,7 @@ overrides: Port: 9095, }, StorageDirectory: "/tmp/loki", - ObjectStorage: ObjectStorage{ + S3ObjectStorage: &S3ObjectStorage{ Endpoint: "http://test.default.svc.cluster.local.:9000", Region: "us-east", Buckets: "loki", @@ -232,9 +233,17 @@ chunk_store_config: enable_fifocache: true fifocache: max_size_bytes: 500MB +common: + storage: + s3: + s3: http://test.default.svc.cluster.local.:9000 + bucketnames: loki + region: us-east + access_key_id: test + secret_access_key: test123 + s3forcepathstyle: true compactor: compaction_interval: 2h - shared_store: s3 working_directory: /tmp/loki/compactor frontend: tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 @@ -362,13 +371,6 @@ storage_config: shared_store: s3 index_gateway_client: server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 - aws: - s3: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: test - secret_access_key: test123 - s3forcepathstyle: true tracing: enabled: false ` @@ -434,7 +436,7 @@ overrides: Port: 9095, }, StorageDirectory: "/tmp/loki", - ObjectStorage: ObjectStorage{ + S3ObjectStorage: &S3ObjectStorage{ Endpoint: "http://test.default.svc.cluster.local.:9000", Region: "us-east", Buckets: "loki", @@ -495,7 +497,7 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { Port: 9095, }, StorageDirectory: "/tmp/loki", - ObjectStorage: ObjectStorage{ + S3ObjectStorage: &S3ObjectStorage{ Endpoint: "http://test.default.svc.cluster.local.:9000", Region: "us-east", Buckets: "loki", From cde4a5744a15aa1bf5cfd41cbc15de3f521b9fd0 Mon Sep 17 00:00:00 2001 From: Gerard Vanloo Date: Fri, 18 Feb 2022 17:04:57 -0500 Subject: [PATCH 3/5] Documentation update --- operator/api/v1beta1/zz_generated.deepcopy.go | 1 - operator/docs/hack_operator_make_run.md | 30 +++++++++---------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/operator/api/v1beta1/zz_generated.deepcopy.go b/operator/api/v1beta1/zz_generated.deepcopy.go index 037f360de672a..81108bddc75f3 100644 --- a/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/operator/api/v1beta1/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. diff --git a/operator/docs/hack_operator_make_run.md b/operator/docs/hack_operator_make_run.md index 958cf38fcb491..f024700f2f4dd 100644 --- a/operator/docs/hack_operator_make_run.md +++ b/operator/docs/hack_operator_make_run.md @@ -54,7 +54,7 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ```console kubectl rollout status deployment/ ``` - + where `` is the name of the deployment and can be found using: ```console @@ -62,19 +62,19 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ``` Confirm that all are up and running for `statefulsets` using: - + ```console kubectl rollout status statefulset/ ``` - + where `` is the name of the statefulset and can be found using: - + ```console kubectl get statefulsets ``` - + * If you make some changes to the operator's code, then just stop the operator locally using `CTRL + C`, update the code and rerun the operator locally: - + ```console make run ``` @@ -92,7 +92,7 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ```console make uninstall ``` - + * Cleanup the minio deployment using: ```console @@ -120,7 +120,7 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ```console kubectl get crd lokistacks.loki.grafana.com ``` - + * Create the `openshift-logging` namespace in the cluster: ```console @@ -130,17 +130,15 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev * Now you need to create a storage secret for the operator. This can be done using: ```console - make olm-deploy-example-storage-secret + ./hack/deploy-aws-storage-secret.sh ``` - OR + This secret will be available in `openshift-logging` namespace. You can check the `hack/deploy-aws-storage-secret.sh` file to check the content of the secret. By default, the script will pull credential information using the `aws` cli. However, these values can be overwritten. For example: ```console - ./hack/deploy-example-secret.sh openshift-logging + REGION=us-west-1 ./hack/deploy-aws-storage-secret.sh ``` - This secret will be available in openshift-logging namespace. You can check the `hack/deploy-example-secret.sh` file to check the content of the secret. - * Once the object storage secret is created, you can now create a LokiStack instance: ```console @@ -178,7 +176,7 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev ```console kubectl -n openshift-logging get statefulsets ``` - + * If you want `lokistack-gateway` component [1] to be deployed then you need to create a gateway secret [2] for the operator. This can be done using: ```code @@ -187,13 +185,13 @@ _Note:_ This is helpful when you don't want to deploy the Loki Operator image ev --from-literal=clientSecret="" \ --from-literal=issuerCAPath="" ``` - + * Now create a LokiStack instance using: ```console kubectl -n openshift-logging apply -f hack/lokistack_gateway_dev.yaml ``` - + * Edit the [main file](https://github.com/grafana/loki/blob/master/operator/main.go) to set the flag values to `true` and rerun the operator using: ```console From ce42f925580560d38f823c4f14ab449a00d80086 Mon Sep 17 00:00:00 2001 From: Gerard Vanloo Date: Wed, 23 Feb 2022 10:34:31 -0500 Subject: [PATCH 4/5] Addressing comments --- operator/api/v1beta1/lokistack_types.go | 14 +-- .../loki-operator.clusterserviceversion.yaml | 10 +- .../loki.grafana.com_lokistacks.yaml | 10 +- operator/cmd/loki-broker/main.go | 25 ++--- .../bases/loki.grafana.com_lokistacks.yaml | 10 +- .../loki-operator.clusterserviceversion.yaml | 10 +- operator/hack/lokistack_dev.yaml | 2 +- operator/hack/lokistack_gateway_dev.yaml | 2 +- operator/hack/lokistack_gateway_ocp.yaml | 2 +- .../handlers/internal/secrets/secrets.go | 95 ++++++++----------- .../handlers/lokistack_create_or_update.go | 8 +- operator/internal/manifests/config.go | 50 +--------- .../manifests/internal/config/build_test.go | 49 ++++++---- .../internal/config/loki-config.yaml | 59 ++++++------ .../manifests/internal/config/options.go | 65 +++---------- .../internal/manifests/node_placement_test.go | 9 +- operator/internal/manifests/options.go | 51 +--------- .../internal/manifests/storage/options.go | 50 ++++++++++ 18 files changed, 216 insertions(+), 305 deletions(-) create mode 100644 operator/internal/manifests/storage/options.go diff --git a/operator/api/v1beta1/lokistack_types.go b/operator/api/v1beta1/lokistack_types.go index d4b74dbeae83b..02b53cafc7468 100644 --- a/operator/api/v1beta1/lokistack_types.go +++ b/operator/api/v1beta1/lokistack_types.go @@ -309,30 +309,30 @@ type LokiTemplateSpec struct { // ObjectStorageSecretType defines the type of storage which can be used with the Loki cluster. // -// +kubebuilder:validation:Enum=Azure;GCS;Swift;S3 +// +kubebuilder:validation:Enum=azure;gcs;s3;swift type ObjectStorageSecretType string const ( // ObjectStorageSecretAzure when using Azure for Loki storage - ObjectStorageSecretAzure ObjectStorageSecretType = "Azure" + ObjectStorageSecretAzure ObjectStorageSecretType = "azure" // ObjectStorageSecretGCS when using GCS for Loki storage - ObjectStorageSecretGCS ObjectStorageSecretType = "GCS" + ObjectStorageSecretGCS ObjectStorageSecretType = "gcs" // ObjectStorageSecretS3 when using S3 for Loki storage - ObjectStorageSecretS3 ObjectStorageSecretType = "S3" + ObjectStorageSecretS3 ObjectStorageSecretType = "s3" // ObjectStorageSecretSwift when using Swift for Loki storage - ObjectStorageSecretSwift ObjectStorageSecretType = "Swift" + ObjectStorageSecretSwift ObjectStorageSecretType = "swift" ) // ObjectStorageSecretSpec is a secret reference containing name only, no namespace. type ObjectStorageSecretSpec struct { - // Type of object storage that needs to be configured + // Type of object storage that should be used // // +required // +kubebuilder:validation:Required - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:select:Azure","urn:alm:descriptor:com.tectonic.ui:select:GCS","urn:alm:descriptor:com.tectonic.ui:select:S3","urn:alm:descriptor:com.tectonic.ui:select:Swift"},displayName="Object Storage Secret Type" + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:select:azure","urn:alm:descriptor:com.tectonic.ui:select:gcs","urn:alm:descriptor:com.tectonic.ui:select:s3","urn:alm:descriptor:com.tectonic.ui:select:swift"},displayName="Object Storage Secret Type" Type ObjectStorageSecretType `json:"type"` // Name of a secret in the namespace configured for object storage secrets. diff --git a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml index 44cbcd95b70e7..ec9541ca95af0 100644 --- a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml @@ -244,14 +244,14 @@ spec: path: storage.secret.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret - - description: Type of object storage that needs to be configured + - description: Type of object storage that should be used displayName: Object Storage Secret Type path: storage.secret.type x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:select:Azure - - urn:alm:descriptor:com.tectonic.ui:select:GCS - - urn:alm:descriptor:com.tectonic.ui:select:S3 - - urn:alm:descriptor:com.tectonic.ui:select:Swift + - urn:alm:descriptor:com.tectonic.ui:select:azure + - urn:alm:descriptor:com.tectonic.ui:select:gcs + - urn:alm:descriptor:com.tectonic.ui:select:s3 + - urn:alm:descriptor:com.tectonic.ui:select:swift - description: Storage class name defines the storage class for ingester/querier PVCs. displayName: Storage Class Name diff --git a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml index 1ed0d8ff24765..852bb493de1ca 100644 --- a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml @@ -221,12 +221,12 @@ spec: for object storage secrets. type: string type: - description: Type of object storage that needs to be configured + description: Type of object storage that should be used enum: - - Azure - - GCS - - Swift - - S3 + - azure + - gcs + - s3 + - swift type: string required: - name diff --git a/operator/cmd/loki-broker/main.go b/operator/cmd/loki-broker/main.go index ef6df099e78f4..76b4e75beba36 100644 --- a/operator/cmd/loki-broker/main.go +++ b/operator/cmd/loki-broker/main.go @@ -11,6 +11,7 @@ import ( "github.com/ViaQ/logerr/log" "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/manifests" + "github.com/grafana/loki/operator/internal/manifests/storage" "sigs.k8s.io/yaml" ) @@ -21,7 +22,7 @@ type config struct { Image string featureFlags manifests.FeatureFlags - objectStorage manifests.ObjectStorage + objectStorage storage.Options crFilepath string writeToDir string @@ -39,14 +40,14 @@ func (c *config) registerFlags(f *flag.FlagSet) { f.BoolVar(&c.featureFlags.EnableTLSServiceMonitorConfig, "with-tls-service-monitors", false, "Enable TLS endpoint for service monitors.") f.BoolVar(&c.featureFlags.EnableGateway, "with-lokistack-gateway", false, "Enables the manifest creation for the entire lokistack-gateway.") // Object storage options - c.objectStorage = manifests.ObjectStorage{ - S3: &manifests.S3Config{}, + c.objectStorage = storage.Options{ + S3: &storage.S3StorageConfig{}, } - f.StringVar(&c.objectStorage.S3.Endpoint, "object-storage.endpoint", "", "The S3 endpoint location.") - f.StringVar(&c.objectStorage.S3.Buckets, "object-storage.buckets", "", "A comma-separated list of S3 buckets.") - f.StringVar(&c.objectStorage.S3.Region, "object-storage.region", "", "An S3 region.") - f.StringVar(&c.objectStorage.S3.AccessKeyID, "object-storage.access-key-id", "", "The access key id for S3.") - f.StringVar(&c.objectStorage.S3.AccessKeySecret, "object-storage.access-key-secret", "", "The access key secret for S3.") + f.StringVar(&c.objectStorage.S3.Endpoint, "object-storage.s3.endpoint", "", "The S3 endpoint location.") + f.StringVar(&c.objectStorage.S3.Buckets, "object-storage.s3.buckets", "", "A comma-separated list of S3 buckets.") + f.StringVar(&c.objectStorage.S3.Region, "object-storage.s3.region", "", "An S3 region.") + f.StringVar(&c.objectStorage.S3.AccessKeyID, "object-storage.s3.access-key-id", "", "The access key id for S3.") + f.StringVar(&c.objectStorage.S3.AccessKeySecret, "object-storage.s3.access-key-secret", "", "The access key secret for S3.") // Input and output file/dir options f.StringVar(&c.crFilepath, "custom-resource.path", "", "Path to a custom resource YAML file.") f.StringVar(&c.writeToDir, "output.write-dir", "", "write each file to the specified directory.") @@ -67,19 +68,19 @@ func (c *config) validateFlags() { } // Validate manifests.objectStorage if cfg.objectStorage.S3.Endpoint == "" { - log.Info("-object.storage.endpoint flag is required") + log.Info("-object-storage.s3.endpoint flag is required") os.Exit(1) } if cfg.objectStorage.S3.Buckets == "" { - log.Info("-object.storage.buckets flag is required") + log.Info("-object-storage.s3.buckets flag is required") os.Exit(1) } if cfg.objectStorage.S3.AccessKeyID == "" { - log.Info("-object.storage.access.key.id flag is required") + log.Info("-object-storage.s3.access.key.id flag is required") os.Exit(1) } if cfg.objectStorage.S3.AccessKeySecret == "" { - log.Info("-object.storage.access.key.secret flag is required") + log.Info("-object-storage.s3.access.key.secret flag is required") os.Exit(1) } } diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 77b9aa2625bd8..ae454466ee07f 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -174,12 +174,12 @@ spec: description: Name of a secret in the namespace configured for object storage secrets. type: string type: - description: Type of object storage that needs to be configured + description: Type of object storage that should be used enum: - - Azure - - GCS - - Swift - - S3 + - azure + - gcs + - s3 + - swift type: string required: - name diff --git a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml index b3cfd76996524..5628e508a6120 100644 --- a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml @@ -223,14 +223,14 @@ spec: path: storage.secret.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret - - description: Type of object storage that needs to be configured + - description: Type of object storage that should be used displayName: Object Storage Secret Type path: storage.secret.type x-descriptors: - - urn:alm:descriptor:com.tectonic.ui:select:Azure - - urn:alm:descriptor:com.tectonic.ui:select:GCS - - urn:alm:descriptor:com.tectonic.ui:select:S3 - - urn:alm:descriptor:com.tectonic.ui:select:Swift + - urn:alm:descriptor:com.tectonic.ui:select:azure + - urn:alm:descriptor:com.tectonic.ui:select:gcs + - urn:alm:descriptor:com.tectonic.ui:select:s3 + - urn:alm:descriptor:com.tectonic.ui:select:swift - description: Storage class name defines the storage class for ingester/querier PVCs. displayName: Storage Class Name diff --git a/operator/hack/lokistack_dev.yaml b/operator/hack/lokistack_dev.yaml index b640a4adb2243..568c7d3889675 100644 --- a/operator/hack/lokistack_dev.yaml +++ b/operator/hack/lokistack_dev.yaml @@ -8,5 +8,5 @@ spec: storage: secret: name: test - type: S3 + type: s3 storageClassName: standard diff --git a/operator/hack/lokistack_gateway_dev.yaml b/operator/hack/lokistack_gateway_dev.yaml index 1202cecc05cb4..7f818196ab05d 100644 --- a/operator/hack/lokistack_gateway_dev.yaml +++ b/operator/hack/lokistack_gateway_dev.yaml @@ -8,7 +8,7 @@ spec: storage: secret: name: test - type: S3 + type: s3 storageClassName: gp2 tenants: mode: static diff --git a/operator/hack/lokistack_gateway_ocp.yaml b/operator/hack/lokistack_gateway_ocp.yaml index c15e643385995..495980c1405d4 100644 --- a/operator/hack/lokistack_gateway_ocp.yaml +++ b/operator/hack/lokistack_gateway_ocp.yaml @@ -8,7 +8,7 @@ spec: storage: secret: name: test - type: S3 + type: s3 storageClassName: gp2 tenants: mode: openshift-logging diff --git a/operator/internal/handlers/internal/secrets/secrets.go b/operator/internal/handlers/internal/secrets/secrets.go index a551e82eada92..898cd18577d23 100644 --- a/operator/internal/handlers/internal/secrets/secrets.go +++ b/operator/internal/handlers/internal/secrets/secrets.go @@ -4,34 +4,11 @@ import ( "github.com/ViaQ/logerr/kverrors" lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/manifests" + "github.com/grafana/loki/operator/internal/manifests/storage" corev1 "k8s.io/api/core/v1" ) -// ExtractStorageSecret reads a k8s secret into a manifest object storage struct if valid. -func ExtractStorageSecret(s *corev1.Secret, secretType lokiv1beta1.ObjectStorageSecretType) (*manifests.ObjectStorage, error) { - var err error - storage := manifests.ObjectStorage{} - - switch secretType { - case lokiv1beta1.ObjectStorageSecretAzure: - storage.Azure, err = extractAzureConfigSecret(s) - case lokiv1beta1.ObjectStorageSecretGCS: - storage.GCS, err = extractGCSConfigSecret(s) - case lokiv1beta1.ObjectStorageSecretS3: - storage.S3, err = extractS3ConfigSecret(s) - case lokiv1beta1.ObjectStorageSecretSwift: - storage.Swift, err = extractSwiftConfigSecret(s) - default: - return nil, kverrors.New("unknown secret type", "type", secretType) - } - - if err != nil { - return nil, err - } - return &storage, nil -} - // ExtractGatewaySecret reads a k8s secret into a manifest tenant secret struct if valid. func ExtractGatewaySecret(s *corev1.Secret, tenantName string) (*manifests.TenantSecrets, error) { // Extract and validate mandatory fields @@ -56,7 +33,31 @@ func ExtractGatewaySecret(s *corev1.Secret, tenantName string) (*manifests.Tenan }, nil } -func extractAzureConfigSecret(s *corev1.Secret) (*manifests.AzureConfig, error) { +// ExtractStorageSecret reads a k8s secret into a manifest object storage struct if valid. +func ExtractStorageSecret(s *corev1.Secret, secretType lokiv1beta1.ObjectStorageSecretType) (*storage.Options, error) { + var err error + storage := storage.Options{} + + switch secretType { + case lokiv1beta1.ObjectStorageSecretAzure: + storage.Azure, err = extractAzureConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretGCS: + storage.GCS, err = extractGCSConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretS3: + storage.S3, err = extractS3ConfigSecret(s) + case lokiv1beta1.ObjectStorageSecretSwift: + storage.Swift, err = extractSwiftConfigSecret(s) + default: + return nil, kverrors.New("unknown secret type", "type", secretType) + } + + if err != nil { + return nil, err + } + return &storage, nil +} + +func extractAzureConfigSecret(s *corev1.Secret) (*storage.AzureStorageConfig, error) { // Extract and validate mandatory fields env, ok := s.Data["environment"] if !ok { @@ -75,7 +76,7 @@ func extractAzureConfigSecret(s *corev1.Secret) (*manifests.AzureConfig, error) return nil, kverrors.New("missing secret field", "field", "account_key") } - return &manifests.AzureConfig{ + return &storage.AzureStorageConfig{ Env: string(env), Container: string(container), AccountName: string(name), @@ -83,19 +84,19 @@ func extractAzureConfigSecret(s *corev1.Secret) (*manifests.AzureConfig, error) }, nil } -func extractGCSConfigSecret(s *corev1.Secret) (*manifests.GCSConfig, error) { +func extractGCSConfigSecret(s *corev1.Secret) (*storage.GCSStorageConfig, error) { // Extract and validate mandatory fields bucket, ok := s.Data["bucketname"] if !ok { return nil, kverrors.New("missing secret field", "field", "bucketname") } - return &manifests.GCSConfig{ + return &storage.GCSStorageConfig{ Bucket: string(bucket), }, nil } -func extractS3ConfigSecret(s *corev1.Secret) (*manifests.S3Config, error) { +func extractS3ConfigSecret(s *corev1.Secret) (*storage.S3StorageConfig, error) { // Extract and validate mandatory fields endpoint, ok := s.Data["endpoint"] if !ok { @@ -116,12 +117,9 @@ func extractS3ConfigSecret(s *corev1.Secret) (*manifests.S3Config, error) { } // Extract and validate optional fields - region, ok := s.Data["region"] - if !ok { - region = []byte("") - } + region := s.Data["region"] - return &manifests.S3Config{ + return &storage.S3StorageConfig{ Endpoint: string(endpoint), Buckets: string(buckets), AccessKeyID: string(id), @@ -130,7 +128,7 @@ func extractS3ConfigSecret(s *corev1.Secret) (*manifests.S3Config, error) { }, nil } -func extractSwiftConfigSecret(s *corev1.Secret) (*manifests.SwiftConfig, error) { +func extractSwiftConfigSecret(s *corev1.Secret) (*storage.SwiftStorageConfig, error) { // Extract and validate mandatory fields url, ok := s.Data["auth_url"] if !ok { @@ -170,28 +168,13 @@ func extractSwiftConfigSecret(s *corev1.Secret) (*manifests.SwiftConfig, error) } // Extract and validate optional fields - projectID, ok := s.Data["project_id"] - if !ok { - projectID = []byte("") - } - projectName, ok := s.Data["project_name"] - if !ok { - projectName = []byte("") - } - projectDomainID, ok := s.Data["project_domain_id"] - if !ok { - projectDomainID = []byte("") - } - projectDomainName, ok := s.Data["project_domain_name"] - if !ok { - projectDomainName = []byte("") - } - region, ok := s.Data["region"] - if !ok { - region = []byte("") - } + projectID := s.Data["project_id"] + projectName := s.Data["project_name"] + projectDomainID := s.Data["project_domain_id"] + projectDomainName := s.Data["project_domain_name"] + region := s.Data["region"] - return &manifests.SwiftConfig{ + return &storage.SwiftStorageConfig{ AuthURL: string(url), Username: string(username), UserDomainName: string(userDomainName), diff --git a/operator/internal/handlers/lokistack_create_or_update.go b/operator/internal/handlers/lokistack_create_or_update.go index 3fb781a996270..cfe6230b7cf3f 100644 --- a/operator/internal/handlers/lokistack_create_or_update.go +++ b/operator/internal/handlers/lokistack_create_or_update.go @@ -49,19 +49,19 @@ func CreateOrUpdateLokiStack(ctx context.Context, req ctrl.Request, k k8s.Client gwImg = manifests.DefaultLokiStackGatewayImage } - var s3secret corev1.Secret + var storageSecret corev1.Secret key := client.ObjectKey{Name: stack.Spec.Storage.Secret.Name, Namespace: stack.Namespace} - if err := k.Get(ctx, key, &s3secret); err != nil { + if err := k.Get(ctx, key, &storageSecret); err != nil { if apierrors.IsNotFound(err) { return status.SetDegradedCondition(ctx, k, req, "Missing object storage secret", lokiv1beta1.ReasonMissingObjectStorageSecret, ) } - return kverrors.Wrap(err, "failed to lookup lokistack s3 secret", "name", key) + return kverrors.Wrap(err, "failed to lookup lokistack storage secret", "name", key) } - storage, err := secrets.ExtractStorageSecret(&s3secret, stack.Spec.Storage.Secret.Type) + storage, err := secrets.ExtractStorageSecret(&storageSecret, stack.Spec.Storage.Secret.Type) if err != nil { return status.SetDegradedCondition(ctx, k, req, "Invalid object storage secret contents", diff --git a/operator/internal/manifests/config.go b/operator/internal/manifests/config.go index 04e547e28010b..0ac9ae169f5a3 100644 --- a/operator/internal/manifests/config.go +++ b/operator/internal/manifests/config.go @@ -42,49 +42,6 @@ func LokiConfigMap(opt Options) (*corev1.ConfigMap, string, error) { // ConfigOptions converts Options to config.Options func ConfigOptions(opt Options) config.Options { - var azureStorage *config.AzureObjectStorage - var gcsStorage *config.GCSObjectStorage - var s3Storage *config.S3ObjectStorage - var swiftStorage *config.SwiftObjectStorage - - if opt.ObjectStorage.Azure != nil { - azureStorage = &config.AzureObjectStorage{ - Env: opt.ObjectStorage.Azure.Env, - Container: opt.ObjectStorage.Azure.Container, - AccountName: opt.ObjectStorage.Azure.AccountName, - AccountKey: opt.ObjectStorage.Azure.AccountKey, - } - } else if opt.ObjectStorage.GCS != nil { - gcsStorage = &config.GCSObjectStorage{ - Bucket: opt.ObjectStorage.GCS.Bucket, - } - } else if opt.ObjectStorage.S3 != nil { - s3Storage = &config.S3ObjectStorage{ - Endpoint: opt.ObjectStorage.S3.Endpoint, - Region: opt.ObjectStorage.S3.Region, - Buckets: opt.ObjectStorage.S3.Buckets, - AccessKeyID: opt.ObjectStorage.S3.AccessKeyID, - AccessKeySecret: opt.ObjectStorage.S3.AccessKeySecret, - } - } else if opt.ObjectStorage.Swift != nil { - swiftStorage = &config.SwiftObjectStorage{ - AuthURL: opt.ObjectStorage.Swift.AuthURL, - Username: opt.ObjectStorage.Swift.Username, - UserDomainName: opt.ObjectStorage.Swift.UserDomainName, - UserDomainID: opt.ObjectStorage.Swift.UserDomainID, - UserID: opt.ObjectStorage.Swift.UserID, - Password: opt.ObjectStorage.Swift.Password, - DomainID: opt.ObjectStorage.Swift.DomainID, - DomainName: opt.ObjectStorage.Swift.DomainName, - ProjectID: opt.ObjectStorage.Swift.ProjectID, - ProjectName: opt.ObjectStorage.Swift.ProjectName, - ProjectDomainID: opt.ObjectStorage.Swift.ProjectDomainID, - ProjectDomainName: opt.ObjectStorage.Swift.ProjectDomainName, - Region: opt.ObjectStorage.Swift.Region, - Container: opt.ObjectStorage.Swift.Container, - } - } - return config.Options{ Stack: opt.Stack, Namespace: opt.Namespace, @@ -105,11 +62,7 @@ func ConfigOptions(opt Options) config.Options { FQDN: fqdn(NewIndexGatewayGRPCService(opt).GetName(), opt.Namespace), Port: grpcPort, }, - StorageDirectory: dataDirectory, - AzureObjectStorage: azureStorage, - GCSObjectStorage: gcsStorage, - S3ObjectStorage: s3Storage, - SwiftObjectStorage: swiftStorage, + StorageDirectory: dataDirectory, QueryParallelism: config.Parallelism{ QuerierCPULimits: opt.ResourceRequirements.Querier.Requests.Cpu().Value(), QueryFrontendReplicas: opt.Stack.Template.QueryFrontend.Replicas, @@ -118,6 +71,7 @@ func ConfigOptions(opt Options) config.Options { Directory: walDirectory, IngesterMemoryRequest: opt.ResourceRequirements.Ingester.Requests.Memory().Value(), }, + ObjectStorage: opt.ObjectStorage, } } diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 478cee509f1ce..832c4c7e89cc6 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -4,6 +4,7 @@ import ( "testing" lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" + "github.com/grafana/loki/operator/internal/manifests/storage" "github.com/stretchr/testify/require" ) @@ -202,13 +203,6 @@ overrides: Port: 9095, }, StorageDirectory: "/tmp/loki", - S3ObjectStorage: &S3ObjectStorage{ - Endpoint: "http://test.default.svc.cluster.local.:9000", - Region: "us-east", - Buckets: "loki", - AccessKeyID: "test", - AccessKeySecret: "test123", - }, QueryParallelism: Parallelism{ QuerierCPULimits: 2, QueryFrontendReplicas: 2, @@ -217,6 +211,15 @@ overrides: Directory: "/tmp/wal", IngesterMemoryRequest: 5000, }, + ObjectStorage: storage.Options{ + S3: &storage.S3StorageConfig{ + Endpoint: "http://test.default.svc.cluster.local.:9000", + Region: "us-east", + Buckets: "loki", + AccessKeyID: "test", + AccessKeySecret: "test123", + }, + }, } cfg, rCfg, err := Build(opts) require.NoError(t, err) @@ -436,13 +439,6 @@ overrides: Port: 9095, }, StorageDirectory: "/tmp/loki", - S3ObjectStorage: &S3ObjectStorage{ - Endpoint: "http://test.default.svc.cluster.local.:9000", - Region: "us-east", - Buckets: "loki", - AccessKeyID: "test", - AccessKeySecret: "test123", - }, QueryParallelism: Parallelism{ QuerierCPULimits: 2, QueryFrontendReplicas: 2, @@ -451,6 +447,15 @@ overrides: Directory: "/tmp/wal", IngesterMemoryRequest: 5000, }, + ObjectStorage: storage.Options{ + S3: &storage.S3StorageConfig{ + Endpoint: "http://test.default.svc.cluster.local.:9000", + Region: "us-east", + Buckets: "loki", + AccessKeyID: "test", + AccessKeySecret: "test123", + }, + }, } cfg, rCfg, err := Build(opts) require.NoError(t, err) @@ -497,13 +502,6 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { Port: 9095, }, StorageDirectory: "/tmp/loki", - S3ObjectStorage: &S3ObjectStorage{ - Endpoint: "http://test.default.svc.cluster.local.:9000", - Region: "us-east", - Buckets: "loki", - AccessKeyID: "test", - AccessKeySecret: "test123", - }, QueryParallelism: Parallelism{ QuerierCPULimits: 2, QueryFrontendReplicas: 2, @@ -512,6 +510,15 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { Directory: "/tmp/wal", IngesterMemoryRequest: 5000, }, + ObjectStorage: storage.Options{ + S3: &storage.S3StorageConfig{ + Endpoint: "http://test.default.svc.cluster.local.:9000", + Region: "us-east", + Buckets: "loki", + AccessKeyID: "test", + AccessKeySecret: "test123", + }, + }, } cfg, rCfg, err := Build(opts) require.Error(t, err) diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index 36b23ba1bd5d4..22e009679cfdb 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -7,39 +7,42 @@ chunk_store_config: max_size_bytes: 500MB common: storage: - {{ if .AzureObjectStorage }} + {{ with .ObjectStorage.Azure }} azure: - environment: {{ .AzureObjectStorage.Env }} - container_name: {{ .AzureObjectStorage.Container }} - account_name: {{ .AzureObjectStorage.AccountName }} - account_key: {{ .AzureObjectStorage.AccountKey }} - {{ else if .GCSObjectStorage }} + environment: {{ .Env }} + container_name: {{ .Container }} + account_name: {{ .AccountName }} + account_key: {{ .AccountKey }} + {{ end }} + {{ with .ObjectStorage.GCS }} gcs: - bucket_name: {{ .GCSObjectStorage.Bucket }} - {{ else if .S3ObjectStorage }} + bucket_name: {{ .Bucket }} + {{ end }} + {{ with .ObjectStorage.S3 }} s3: - s3: {{ .S3ObjectStorage.Endpoint }} - bucketnames: {{ .S3ObjectStorage.Buckets }} - region: {{ .S3ObjectStorage.Region }} - access_key_id: {{ .S3ObjectStorage.AccessKeyID }} - secret_access_key: {{ .S3ObjectStorage.AccessKeySecret }} + s3: {{ .Endpoint }} + bucketnames: {{ .Buckets }} + region: {{ .Region }} + access_key_id: {{ .AccessKeyID }} + secret_access_key: {{ .AccessKeySecret }} s3forcepathstyle: true - {{ else if .SwiftObjectStorage }} + {{ end }} + {{ with .ObjectStorage.Swift }} swift: - auth_url: {{ .SwiftObjectStorage.AuthURL }} - username: {{ .SwiftObjectStorage.Username }} - user_domain_name: {{ .SwiftObjectStorage.UserDomainName }} - user_domain_id: {{ .SwiftObjectStorage.UserDomainID }} - user_id: {{ .SwiftObjectStorage.UserID }} - password: {{ .SwiftObjectStorage.Password }} - domain_id: {{ .SwiftObjectStorage.DomainID }} - domain_name: {{ .SwiftObjectStorage.DomainName }} - project_id: {{ .SwiftObjectStorage.ProjectID }} - project_name: {{ .SwiftObjectStorage.ProjectName }} - project_domain_id: {{ .SwiftObjectStorage.ProjectDomainID }} - project_domain_name: {{ .SwiftObjectStorage.ProjectDomainName }} - region_name: {{ .SwiftObjectStorage.Region }} - container_name: {{ .SwiftObjectStorage.Container }} + auth_url: {{ .AuthURL }} + username: {{ .Username }} + user_domain_name: {{ .UserDomainName }} + user_domain_id: {{ .UserDomainID }} + user_id: {{ .UserID }} + password: {{ .Password }} + domain_id: {{ .DomainID }} + domain_name: {{ .DomainName }} + project_id: {{ .ProjectID }} + project_name: {{ .ProjectName }} + project_domain_id: {{ .ProjectDomainID }} + project_domain_name: {{ .ProjectDomainName }} + region_name: {{ .Region }} + container_name: {{ .Container }} {{ end }} compactor: compaction_interval: 2h diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index 7a90f9c96d751..a0992cf52b97e 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -5,25 +5,24 @@ import ( "math" lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" + "github.com/grafana/loki/operator/internal/manifests/storage" ) // Options is used to render the loki-config.yaml file template type Options struct { Stack lokiv1beta1.LokiStackSpec - Namespace string - Name string - FrontendWorker Address - GossipRing Address - Querier Address - IndexGateway Address - StorageDirectory string - AzureObjectStorage *AzureObjectStorage - GCSObjectStorage *GCSObjectStorage - S3ObjectStorage *S3ObjectStorage - SwiftObjectStorage *SwiftObjectStorage - QueryParallelism Parallelism - WriteAheadLog WriteAheadLog + Namespace string + Name string + FrontendWorker Address + GossipRing Address + Querier Address + IndexGateway Address + StorageDirectory string + QueryParallelism Parallelism + WriteAheadLog WriteAheadLog + + ObjectStorage storage.Options } // Address FQDN and port for a k8s service. @@ -34,46 +33,6 @@ type Address struct { Port int } -// AzureObjectStorage for Azure storage config -type AzureObjectStorage struct { - Env string - Container string - AccountName string - AccountKey string -} - -// GCSObjectStorage for GCS storage config -type GCSObjectStorage struct { - Bucket string -} - -// S3ObjectStorage for S3 storage config -type S3ObjectStorage struct { - Endpoint string - Region string - Buckets string - AccessKeyID string - AccessKeySecret string -} - -// SwiftObjectStorage for Swift storage config -type SwiftObjectStorage struct { - AuthURL string - Username string - UserDomainName string - UserDomainID string - UserID string - Password string - DomainID string - DomainName string - ProjectID string - ProjectName string - ProjectDomainID string - ProjectDomainName string - Region string - Container string -} - // Parallelism for query processing parallelism // and rate limiting. type Parallelism struct { diff --git a/operator/internal/manifests/node_placement_test.go b/operator/internal/manifests/node_placement_test.go index 399ae9fc2a7c9..857524b2c0c6c 100644 --- a/operator/internal/manifests/node_placement_test.go +++ b/operator/internal/manifests/node_placement_test.go @@ -4,6 +4,7 @@ import ( "testing" lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" + "github.com/grafana/loki/operator/internal/manifests/storage" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" ) @@ -44,7 +45,7 @@ func TestTolerationsAreSetForEachComponent(t *testing.T) { }, }, }, - ObjectStorage: ObjectStorage{}, + ObjectStorage: storage.Options{}, } optsWithoutTolerations := Options{ @@ -70,7 +71,7 @@ func TestTolerationsAreSetForEachComponent(t *testing.T) { }, }, }, - ObjectStorage: ObjectStorage{}, + ObjectStorage: storage.Options{}, } t.Run("distributor", func(t *testing.T) { @@ -135,7 +136,7 @@ func TestNodeSelectorsAreSetForEachComponent(t *testing.T) { }, }, }, - ObjectStorage: ObjectStorage{}, + ObjectStorage: storage.Options{}, } optsWithoutNodeSelectors := Options{ @@ -161,7 +162,7 @@ func TestNodeSelectorsAreSetForEachComponent(t *testing.T) { }, }, }, - ObjectStorage: ObjectStorage{}, + ObjectStorage: storage.Options{}, } t.Run("distributor", func(t *testing.T) { diff --git a/operator/internal/manifests/options.go b/operator/internal/manifests/options.go index 199b9c7362921..054b8e314285b 100644 --- a/operator/internal/manifests/options.go +++ b/operator/internal/manifests/options.go @@ -4,6 +4,7 @@ import ( lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1" "github.com/grafana/loki/operator/internal/manifests/internal" "github.com/grafana/loki/operator/internal/manifests/openshift" + "github.com/grafana/loki/operator/internal/manifests/storage" ) // Options is a set of configuration values to use when building manifests such as resource sizes, etc. @@ -21,61 +22,13 @@ type Options struct { Stack lokiv1beta1.LokiStackSpec ResourceRequirements internal.ComponentResources - ObjectStorage ObjectStorage + ObjectStorage storage.Options OpenShiftOptions openshift.Options TenantSecrets []*TenantSecrets TenantConfigMap map[string]openshift.TenantData } -// ObjectStorage for storage config. -type ObjectStorage struct { - Azure *AzureConfig - GCS *GCSConfig - S3 *S3Config - Swift *SwiftConfig -} - -// AzureConfig for Azure storage config -type AzureConfig struct { - Env string - Container string - AccountName string - AccountKey string -} - -// GCSConfig for GCS storage config -type GCSConfig struct { - Bucket string -} - -// S3Config for S3 storage config -type S3Config struct { - Endpoint string - Region string - Buckets string - AccessKeyID string - AccessKeySecret string -} - -// SwiftConfig for Swift storage config -type SwiftConfig struct { - AuthURL string - Username string - UserDomainName string - UserDomainID string - UserID string - Password string - DomainID string - DomainName string - ProjectID string - ProjectName string - ProjectDomainID string - ProjectDomainName string - Region string - Container string -} - // FeatureFlags contains flags that activate various features type FeatureFlags struct { EnableCertificateSigningService bool diff --git a/operator/internal/manifests/storage/options.go b/operator/internal/manifests/storage/options.go new file mode 100644 index 0000000000000..254fe0a8d853e --- /dev/null +++ b/operator/internal/manifests/storage/options.go @@ -0,0 +1,50 @@ +package storage + +// Options is used to configure Loki to integrate with +// supported object storages. +type Options struct { + Azure *AzureStorageConfig + GCS *GCSStorageConfig + S3 *S3StorageConfig + Swift *SwiftStorageConfig +} + +// AzureStorageConfig for Azure storage config +type AzureStorageConfig struct { + Env string + Container string + AccountName string + AccountKey string +} + +// GCSStorageConfig for GCS storage config +type GCSStorageConfig struct { + Bucket string +} + +// S3StorageConfig for S3 storage config +type S3StorageConfig struct { + Endpoint string + Region string + Buckets string + AccessKeyID string + AccessKeySecret string +} + +// SwiftStorageConfig for Swift storage config +type SwiftStorageConfig struct { + AuthURL string + Username string + UserDomainName string + UserDomainID string + UserID string + Password string + DomainID string + DomainName string + ProjectID string + ProjectName string + ProjectDomainID string + ProjectDomainName string + Region string + Container string +} From f520cdfdcf9b4f68ed165ea30c7a4269f59d50ac Mon Sep 17 00:00:00 2001 From: Gerard Vanloo Date: Thu, 24 Feb 2022 08:30:06 -0500 Subject: [PATCH 5/5] Updated changelog --- operator/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index d86a2754873d3..172368497bce8 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,3 +1,4 @@ ## Main - [4975](https://github.com/grafana/loki/pull/4975) **periklis**: Provide saner default for loki-operator managed chunk_target_size +- [4974](https://github.com/grafana/loki/pull/5432) **Red-GV**: Provide storage configuration for Azure, GCS, and Swift through common_config