From 851219b2d656062ee4f24d551a6df66c6460ad19 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Tue, 24 Sep 2024 16:22:49 +0200 Subject: [PATCH 01/21] Remote Clusters using API Keys --- cmd/manager/main.go | 4 +- config/crds/v1/all-crds.yaml | 52 + ...search.k8s.elastic.co_elasticsearches.yaml | 52 + .../recipes/remoteclusters/elasticsearch.yaml | 81 ++ .../eck-operator-crds/templates/all-crds.yaml | 52 + docs/reference/api-docs.asciidoc | 108 ++ .../elasticsearch/v1/elasticsearch_types.go | 14 + pkg/apis/elasticsearch/v1/fields.go | 9 + pkg/apis/elasticsearch/v1/name.go | 13 + pkg/apis/elasticsearch/v1/remote_cluster.go | 91 ++ .../elasticsearch/v1/zz_generated.deepcopy.go | 136 ++- pkg/controller/common/keystore/resources.go | 3 +- pkg/controller/common/keystore/user_secret.go | 3 + .../certificates/transport/csr.go | 9 + pkg/controller/elasticsearch/client/base.go | 6 +- pkg/controller/elasticsearch/client/client.go | 5 +- pkg/controller/elasticsearch/client/model.go | 20 - .../elasticsearch/client/remote_cluster.go | 163 +++ pkg/controller/elasticsearch/client/v6.go | 16 + pkg/controller/elasticsearch/client/v8.go | 22 + pkg/controller/elasticsearch/driver/driver.go | 58 +- pkg/controller/elasticsearch/network/ports.go | 2 + .../elasticsearch/nodespec/podspec_test.go | 8 +- .../elasticsearch/nodespec/resources.go | 2 +- .../remotecluster/elasticsearch.go | 5 + .../elasticsearch/services/services.go | 70 +- .../elasticsearch/settings/merged_config.go | 33 +- .../settings/merged_config_test.go | 78 +- .../elasticsearch/validation/validations.go | 32 + .../validation/validations_test.go | 98 ++ pkg/controller/remoteca/controller_test.go | 456 -------- pkg/controller/remotecluster/apikey.go | 156 +++ .../{remoteca => remotecluster}/controller.go | 227 +++- .../remotecluster/controller_test.go | 970 ++++++++++++++++++ pkg/controller/remotecluster/fixtures.go | 198 ++++ pkg/controller/remotecluster/keystore.go | 219 ++++ pkg/controller/remotecluster/keystore_test.go | 282 +++++ .../{remoteca => remotecluster}/labels.go | 2 +- .../{remoteca => remotecluster}/rbac.go | 2 +- .../{remoteca => remotecluster}/secret.go | 8 +- .../{remoteca => remotecluster}/watches.go | 70 +- pkg/telemetry/fixtures.go | 5 +- pkg/telemetry/telemetry.go | 6 + pkg/telemetry/telemetry_test.go | 85 +- 44 files changed, 3318 insertions(+), 613 deletions(-) create mode 100644 config/recipes/remoteclusters/elasticsearch.yaml create mode 100644 pkg/apis/elasticsearch/v1/remote_cluster.go create mode 100644 pkg/controller/elasticsearch/client/remote_cluster.go delete mode 100644 pkg/controller/remoteca/controller_test.go create mode 100644 pkg/controller/remotecluster/apikey.go rename pkg/controller/{remoteca => remotecluster}/controller.go (50%) create mode 100644 pkg/controller/remotecluster/controller_test.go create mode 100644 pkg/controller/remotecluster/fixtures.go create mode 100644 pkg/controller/remotecluster/keystore.go create mode 100644 pkg/controller/remotecluster/keystore_test.go rename pkg/controller/{remoteca => remotecluster}/labels.go (98%) rename pkg/controller/{remoteca => remotecluster}/rbac.go (98%) rename pkg/controller/{remoteca => remotecluster}/secret.go (98%) rename pkg/controller/{remoteca => remotecluster}/watches.go (63%) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index eb529f56d9..1f773df265 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -83,7 +83,7 @@ import ( "github.com/elastic/cloud-on-k8s/v2/pkg/controller/logstash" lsvalidation "github.com/elastic/cloud-on-k8s/v2/pkg/controller/logstash/validation" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/maps" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remoteca" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/stackconfigpolicy" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/webhook" "github.com/elastic/cloud-on-k8s/v2/pkg/dev" @@ -899,7 +899,7 @@ func registerControllers(mgr manager.Manager, params operator.Parameters, access name string registerFunc func(manager.Manager, rbac.AccessReviewer, operator.Parameters) error }{ - {name: "RemoteCA", registerFunc: remoteca.Add}, + {name: "RemoteCA", registerFunc: remotecluster.Add}, {name: "APM-ES", registerFunc: associationctl.AddApmES}, {name: "APM-KB", registerFunc: associationctl.AddApmKibana}, {name: "KB-ES", registerFunc: associationctl.AddKibanaES}, diff --git a/config/crds/v1/all-crds.yaml b/config/crds/v1/all-crds.yaml index 645e5e8d49..855a9d90c7 100644 --- a/config/crds/v1/all-crds.yaml +++ b/config/crds/v1/all-crds.yaml @@ -4794,6 +4794,14 @@ spec: type: string type: object type: object + remoteClusterServer: + description: |- + RemoteClusterServer specifies if the remote cluster server must be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. + properties: + enabled: + type: boolean + type: object remoteClusters: description: RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster. @@ -4801,6 +4809,50 @@ spec: description: RemoteCluster declares a remote Elasticsearch cluster connection. properties: + apiKey: + description: 'APIKey can be used to enable remote cluster using + Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + properties: + access: + description: Access is the name of the API Key. It is automatically + generated if not set or empty. + properties: + replication: + properties: + names: + items: + type: string + type: array + required: + - names + type: object + search: + properties: + field_security: + properties: + except: + items: + type: string + type: array + grant: + items: + type: string + type: array + required: + - except + - grant + type: object + names: + items: + type: string + type: array + required: + - names + type: object + type: object + required: + - access + type: object elasticsearchRef: description: ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. diff --git a/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml b/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml index 5a14c70736..2e54c7a0a3 100644 --- a/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml +++ b/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml @@ -9263,6 +9263,14 @@ spec: type: string type: object type: object + remoteClusterServer: + description: |- + RemoteClusterServer specifies if the remote cluster server must be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. + properties: + enabled: + type: boolean + type: object remoteClusters: description: RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster. @@ -9270,6 +9278,50 @@ spec: description: RemoteCluster declares a remote Elasticsearch cluster connection. properties: + apiKey: + description: 'APIKey can be used to enable remote cluster using + Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + properties: + access: + description: Access is the name of the API Key. It is automatically + generated if not set or empty. + properties: + replication: + properties: + names: + items: + type: string + type: array + required: + - names + type: object + search: + properties: + field_security: + properties: + except: + items: + type: string + type: array + grant: + items: + type: string + type: array + required: + - except + - grant + type: object + names: + items: + type: string + type: array + required: + - names + type: object + type: object + required: + - access + type: object elasticsearchRef: description: ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. diff --git a/config/recipes/remoteclusters/elasticsearch.yaml b/config/recipes/remoteclusters/elasticsearch.yaml new file mode 100644 index 0000000000..9b168359c6 --- /dev/null +++ b/config/recipes/remoteclusters/elasticsearch.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ns1 +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: cluster1 + namespace: ns1 +spec: + version: 8.15.0 + remoteClusters: + - name: to-ns2-cluster2 + elasticsearchRef: + name: cluster2 + namespace: ns2 + apiKey: + access: + search: + names: + ## This requires the sample data available at https://kibana_url/app/home#/tutorial_directory/sampleData + - kibana_sample_data_ecommerce + nodeSets: + - name: default + config: + node.store.allow_mmap: false + count: 3 +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana + namespace: ns1 +spec: +# http: +# service: +# spec: +# # expose this cluster Service with a LoadBalancer +# type: LoadBalancer + version: 8.15.0 + count: 1 + elasticsearchRef: + name: "cluster1" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ns2 +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: cluster2 + namespace: ns2 +spec: + version: 8.15.0 + ## Required for this cluster to be accessed using remote cluster API keys. + remoteClusterServer: + enabled: true + nodeSets: + - name: default + config: + node.store.allow_mmap: false + count: 3 +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana + namespace: ns2 +spec: +# http: +# service: +# spec: +# # expose this cluster Service with a LoadBalancer +# type: LoadBalancer + version: 8.15.0 + count: 1 + elasticsearchRef: + name: "cluster2" \ No newline at end of file diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml index ca702d0eda..a70e70c79d 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml @@ -4836,6 +4836,14 @@ spec: type: string type: object type: object + remoteClusterServer: + description: |- + RemoteClusterServer specifies if the remote cluster server must be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. + properties: + enabled: + type: boolean + type: object remoteClusters: description: RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster. @@ -4843,6 +4851,50 @@ spec: description: RemoteCluster declares a remote Elasticsearch cluster connection. properties: + apiKey: + description: 'APIKey can be used to enable remote cluster using + Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + properties: + access: + description: Access is the name of the API Key. It is automatically + generated if not set or empty. + properties: + replication: + properties: + names: + items: + type: string + type: array + required: + - names + type: object + search: + properties: + field_security: + properties: + except: + items: + type: string + type: array + grant: + items: + type: string + type: array + required: + - except + - grant + type: object + names: + items: + type: string + type: array + required: + - names + type: object + type: object + required: + - access + type: object elasticsearchRef: description: ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. diff --git a/docs/reference/api-docs.asciidoc b/docs/reference/api-docs.asciidoc index 31b20b464e..63c0aaca9b 100644 --- a/docs/reference/api-docs.asciidoc +++ b/docs/reference/api-docs.asciidoc @@ -1332,6 +1332,8 @@ ElasticsearchSpec holds the specification of an Elasticsearch cluster. | Field | Description | *`version`* __string__ | Version of Elasticsearch. | *`image`* __string__ | Image is the Elasticsearch Docker image to deploy. +| *`remoteClusterServer`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterserver[$$RemoteClusterServer$$]__ | RemoteClusterServer specifies if the remote cluster server must be enabled. +This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. | *`http`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-httpconfig[$$HTTPConfig$$]__ | HTTP holds HTTP layer settings for Elasticsearch. | *`transport`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-transportconfig[$$TransportConfig$$]__ | Transport holds transport layer settings for Elasticsearch. | *`nodeSets`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-nodeset[$$NodeSet$$] array__ | NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. @@ -1384,6 +1386,24 @@ controller has not yet processed the changes contained in the Elasticsearch spec |=== +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-fieldsecurity"] +=== FieldSecurity + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-search[$$Search$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`grant`* __string array__ | +| *`except`* __string array__ | +|=== + + [id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-filerealmsource"] === FileRealmSource @@ -1497,6 +1517,76 @@ RemoteCluster declares a remote Elasticsearch cluster connection. | *`name`* __string__ | Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters. | *`elasticsearchRef`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-localobjectselector[$$LocalObjectSelector$$]__ | ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. +| *`apiKey`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterapikey[$$RemoteClusterAPIKey$$]__ | APIKey can be used to enable remote cluster using Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html +|=== + + +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterapikey"] +=== RemoteClusterAPIKey + +RemoteClusterAPIKey defines a remote cluster API Key. + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remotecluster[$$RemoteCluster$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`access`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess[$$RemoteClusterAccess$$]__ | Access is the name of the API Key. It is automatically generated if not set or empty. +|=== + + +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess"] +=== RemoteClusterAccess + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterapikey[$$RemoteClusterAPIKey$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`search`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-search[$$Search$$]__ | +| *`replication`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-replication[$$Replication$$]__ | +|=== + + +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterserver"] +=== RemoteClusterServer + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-elasticsearchspec[$$ElasticsearchSpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`enabled`* __boolean__ | +|=== + + +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-replication"] +=== Replication + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess[$$RemoteClusterAccess$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`names`* __string array__ | |=== @@ -1517,6 +1607,24 @@ RoleSource references roles to create in the Elasticsearch cluster. |=== +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-search"] +=== Search + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess[$$RemoteClusterAccess$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`names`* __string array__ | +| *`field_security`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-fieldsecurity[$$FieldSecurity$$]__ | +|=== + + [id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-selfsignedtransportcertificates"] === SelfSignedTransportCertificates diff --git a/pkg/apis/elasticsearch/v1/elasticsearch_types.go b/pkg/apis/elasticsearch/v1/elasticsearch_types.go index f02c349cce..2fb927e433 100644 --- a/pkg/apis/elasticsearch/v1/elasticsearch_types.go +++ b/pkg/apis/elasticsearch/v1/elasticsearch_types.go @@ -82,6 +82,11 @@ type ElasticsearchSpec struct { // Image is the Elasticsearch Docker image to deploy. Image string `json:"image,omitempty"` + // RemoteClusterServer specifies if the remote cluster server must be enabled. + // This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. + // +kubebuilder:validation:Optional + RemoteClusterServer RemoteClusterServer `json:"remoteClusterServer,omitempty"` + // HTTP holds HTTP layer settings for Elasticsearch. // +kubebuilder:validation:Optional HTTP commonv1.HTTPConfig `json:"http,omitempty"` @@ -139,6 +144,11 @@ type ElasticsearchSpec struct { RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` } +type RemoteClusterServer struct { + // +kubebuilder:validation:Optional + Enabled bool `json:"enabled,omitempty"` +} + // VolumeClaimDeletePolicy describes the delete policy for handling PersistentVolumeClaims that hold Elasticsearch data. // Inspired by https://github.com/kubernetes/enhancements/pull/2440 type VolumeClaimDeletePolicy string @@ -206,6 +216,10 @@ type RemoteCluster struct { // ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. ElasticsearchRef commonv1.LocalObjectSelector `json:"elasticsearchRef,omitempty"` + // APIKey can be used to enable remote cluster using Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html + // +kubebuilder:validation:Optional + APIKey *RemoteClusterAPIKey `json:"apiKey,omitempty"` + // TODO: Allow the user to specify some options (transport.compress, transport.ping_schedule) } diff --git a/pkg/apis/elasticsearch/v1/fields.go b/pkg/apis/elasticsearch/v1/fields.go index e5952994f5..fd7a9daa97 100644 --- a/pkg/apis/elasticsearch/v1/fields.go +++ b/pkg/apis/elasticsearch/v1/fields.go @@ -26,6 +26,8 @@ const ( NetworkPublishHost = "network.publish_host" HTTPPublishHost = "http.publish_host" + RemoteClusterEnabled = "remote_cluster_server.enabled" + NodeName = "node.name" PathData = "path.data" @@ -54,6 +56,13 @@ const ( XPackSecurityTransportSslKey = "xpack.security.transport.ssl.key" XPackSecurityTransportSslVerificationMode = "xpack.security.transport.ssl.verification_mode" + XPackSecurityRemoteClusterServerSslKey = "xpack.security.remote_cluster_server.ssl.key" + XPackSecurityRemoteClusterServerSslCertificate = "xpack.security.remote_cluster_server.ssl.certificate" + XPackSecurityRemoteClusterServerSslCertificateAuthorities = "xpack.security.remote_cluster_server.ssl.certificate_authorities" + + XPackSecurityRemoteClusterClientSslKey = "xpack.security.remote_cluster_client.ssl.enabled" + XPackSecurityRemoteClusterClientSslCertificateAuthorities = "xpack.security.remote_cluster_client.ssl.certificate_authorities" + XPackLicenseUploadTypes = "xpack.license.upload.types" // supported >= 7.6.0 used as of 7.8.1 ) diff --git a/pkg/apis/elasticsearch/v1/name.go b/pkg/apis/elasticsearch/v1/name.go index 6d30966df6..58d815e62e 100644 --- a/pkg/apis/elasticsearch/v1/name.go +++ b/pkg/apis/elasticsearch/v1/name.go @@ -23,6 +23,7 @@ const ( policyEsConfigSecretSuffix = "policy-config" //nolint:gosec httpServiceSuffix = "http" internalHTTPServiceSuffix = "internal-http" + remoteClusterServiceSuffix = "remote-cluster" transportServiceSuffix = "transport" elasticUserSecretSuffix = "elastic-user" internalUsersSecretSuffix = "internal-users" @@ -40,6 +41,9 @@ const ( // remoteCaNameSuffix is a suffix for the secret that contains the concatenation of all the remote CAs remoteCaNameSuffix = "remote-ca" + // remoteCredentialsNameSuffix is a suffix for the secret that contains the API keys for the remote clusters. + remoteAPIKeysNameSuffix = "remote-api-keys" + controllerRevisionHashLen = 10 ) @@ -60,6 +64,7 @@ var ( scriptsConfigMapSuffix, statefulSetTransportCertificatesSecretSuffix, remoteCaNameSuffix, + remoteAPIKeysNameSuffix, } ) @@ -136,6 +141,10 @@ func InternalHTTPService(esName string) string { return ESNamer.Suffix(esName, internalHTTPServiceSuffix) } +func RemoteClusterService(esName string) string { + return ESNamer.Suffix(esName, remoteClusterServiceSuffix) +} + func HTTPService(esName string) string { return ESNamer.Suffix(esName, httpServiceSuffix) } @@ -173,6 +182,10 @@ func RemoteCaSecretName(esName string) string { return ESNamer.Suffix(esName, remoteCaNameSuffix) } +func RemoteAPIKeysSecretName(esName string) string { + return ESNamer.Suffix(esName, remoteAPIKeysNameSuffix) +} + func FileSettingsSecretName(esName string) string { return ESNamer.Suffix(esName, fileSettingsSecretSuffix) } diff --git a/pkg/apis/elasticsearch/v1/remote_cluster.go b/pkg/apis/elasticsearch/v1/remote_cluster.go new file mode 100644 index 0000000000..ae71f5592c --- /dev/null +++ b/pkg/apis/elasticsearch/v1/remote_cluster.go @@ -0,0 +1,91 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package v1 + +import ( + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/version" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/optional" +) + +var ( + RemoteClusterAPIKeysMinVersion = version.MinFor(8, 10, 0) +) + +// SupportRemoteClusterAPIKeys returns true if this cluster supports connecting to a remote cluster using API keys. +func (es *Elasticsearch) SupportRemoteClusterAPIKeys() (*optional.Bool, error) { + if es == nil { + return nil, nil + } + if es.Status.Version == "" { + // This cluster is not reconciled yet. + return nil, nil + } + esVersion, err := version.Parse(es.Status.Version) + if err != nil { + return nil, err + } + return optional.NewBool(esVersion.GTE(RemoteClusterAPIKeysMinVersion)), nil +} + +// HasRemoteClusterAPIKey returns true if this cluster is connecting to a remote cluster using API keys. +func (es *Elasticsearch) HasRemoteClusterAPIKey() bool { + if es == nil { + return false + } + for _, remoteCLuster := range es.Spec.RemoteClusters { + if remoteCLuster.APIKey != nil { + return true + } + } + return false +} + +// RemoteClustersCount returns the number of remote clusters using only certificates and API keys. +func (es *Elasticsearch) RemoteClustersCount() (int32, int32) { + if es == nil { + return 0, 0 + } + var withoutAPIKeys, withAPIKeys int32 + for _, remoteCLuster := range es.Spec.RemoteClusters { + if remoteCLuster.APIKey == nil { + withoutAPIKeys++ + continue + } + withAPIKeys++ + } + return withoutAPIKeys, withAPIKeys +} + +// RemoteClusterAPIKey defines a remote cluster API Key. +type RemoteClusterAPIKey struct { + // Access is the name of the API Key. It is automatically generated if not set or empty. + // +kubebuilder:validation:Required + Access RemoteClusterAccess `json:"access,omitempty"` +} + +type RemoteClusterAccess struct { + // +kubebuilder:validation:Optional + Search *Search `json:"search,omitempty"` + // +kubebuilder:validation:Optional + Replication *Replication `json:"replication,omitempty"` +} + +type Search struct { + // +kubebuilder:validation:Required + Names []string `json:"names,omitempty"` + + // +kubebuilder:validation:Optional + FieldSecurity *FieldSecurity `json:"field_security,omitempty"` +} + +type FieldSecurity struct { + Grant []string `json:"grant"` + Except []string `json:"except"` +} + +type Replication struct { + // +kubebuilder:validation:Required + Names []string `json:"names,omitempty"` +} diff --git a/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go b/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go index edb7c903e8..ceecf7bd69 100644 --- a/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go +++ b/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go @@ -255,6 +255,7 @@ func (in *ElasticsearchSettings) DeepCopy() *ElasticsearchSettings { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ElasticsearchSpec) DeepCopyInto(out *ElasticsearchSpec) { *out = *in + out.RemoteClusterServer = in.RemoteClusterServer in.HTTP.DeepCopyInto(&out.HTTP) in.Transport.DeepCopyInto(&out.Transport) if in.NodeSets != nil { @@ -281,7 +282,9 @@ func (in *ElasticsearchSpec) DeepCopyInto(out *ElasticsearchSpec) { if in.RemoteClusters != nil { in, out := &in.RemoteClusters, &out.RemoteClusters *out = make([]RemoteCluster, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } in.Monitoring.DeepCopyInto(&out.Monitoring) if in.RevisionHistoryLimit != nil { @@ -352,6 +355,31 @@ func (in *EsMonitoringAssociation) DeepCopy() *EsMonitoringAssociation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FieldSecurity) DeepCopyInto(out *FieldSecurity) { + *out = *in + if in.Grant != nil { + in, out := &in.Grant, &out.Grant + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Except != nil { + in, out := &in.Except, &out.Except + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FieldSecurity. +func (in *FieldSecurity) DeepCopy() *FieldSecurity { + if in == nil { + return nil + } + out := new(FieldSecurity) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FileRealmSource) DeepCopyInto(out *FileRealmSource) { *out = *in @@ -492,6 +520,11 @@ func (in *NodeSet) DeepCopy() *NodeSet { func (in *RemoteCluster) DeepCopyInto(out *RemoteCluster) { *out = *in out.ElasticsearchRef = in.ElasticsearchRef + if in.APIKey != nil { + in, out := &in.APIKey, &out.APIKey + *out = new(RemoteClusterAPIKey) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteCluster. @@ -504,6 +537,82 @@ func (in *RemoteCluster) DeepCopy() *RemoteCluster { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterAPIKey) DeepCopyInto(out *RemoteClusterAPIKey) { + *out = *in + in.Access.DeepCopyInto(&out.Access) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterAPIKey. +func (in *RemoteClusterAPIKey) DeepCopy() *RemoteClusterAPIKey { + if in == nil { + return nil + } + out := new(RemoteClusterAPIKey) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterAccess) DeepCopyInto(out *RemoteClusterAccess) { + *out = *in + if in.Search != nil { + in, out := &in.Search, &out.Search + *out = new(Search) + (*in).DeepCopyInto(*out) + } + if in.Replication != nil { + in, out := &in.Replication, &out.Replication + *out = new(Replication) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterAccess. +func (in *RemoteClusterAccess) DeepCopy() *RemoteClusterAccess { + if in == nil { + return nil + } + out := new(RemoteClusterAccess) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterServer) DeepCopyInto(out *RemoteClusterServer) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterServer. +func (in *RemoteClusterServer) DeepCopy() *RemoteClusterServer { + if in == nil { + return nil + } + out := new(RemoteClusterServer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Replication) DeepCopyInto(out *Replication) { + *out = *in + if in.Names != nil { + in, out := &in.Names, &out.Names + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Replication. +func (in *Replication) DeepCopy() *Replication { + if in == nil { + return nil + } + out := new(Replication) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RoleSource) DeepCopyInto(out *RoleSource) { *out = *in @@ -520,6 +629,31 @@ func (in *RoleSource) DeepCopy() *RoleSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Search) DeepCopyInto(out *Search) { + *out = *in + if in.Names != nil { + in, out := &in.Names, &out.Names + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.FieldSecurity != nil { + in, out := &in.FieldSecurity, &out.FieldSecurity + *out = new(FieldSecurity) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Search. +func (in *Search) DeepCopy() *Search { + if in == nil { + return nil + } + out := new(Search) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SelfSignedTransportCertificates) DeepCopyInto(out *SelfSignedTransportCertificates) { *out = *in diff --git a/pkg/controller/common/keystore/resources.go b/pkg/controller/common/keystore/resources.go index 0a92f48387..0e07424b69 100644 --- a/pkg/controller/common/keystore/resources.go +++ b/pkg/controller/common/keystore/resources.go @@ -58,9 +58,10 @@ func ReconcileResources( namer name.Namer, labels map[string]string, initContainerParams InitContainerParameters, + additionalSources ...commonv1.NamespacedSecretSource, ) (*Resources, error) { // setup a volume from the user-provided secure settings secret - secretVolume, hash, err := secureSettingsVolume(ctx, r, hasKeystore, labels, namer) + secretVolume, hash, err := secureSettingsVolume(ctx, r, hasKeystore, labels, namer, additionalSources...) if err != nil { return nil, err } diff --git a/pkg/controller/common/keystore/user_secret.go b/pkg/controller/common/keystore/user_secret.go index 8b765d4259..dac92e95a6 100644 --- a/pkg/controller/common/keystore/user_secret.go +++ b/pkg/controller/common/keystore/user_secret.go @@ -44,12 +44,15 @@ func secureSettingsVolume( hasKeystore HasKeystore, labels map[string]string, namer name.Namer, + additionalSources ...commonv1.NamespacedSecretSource, ) (*volume.SecretVolume, string, error) { // setup (or remove) watches for the user-provided secret to reconcile on any change watcher := k8s.ExtractNamespacedName(hasKeystore) // user-provided Secrets referenced in the resource secretSources := WatchedSecretNames(hasKeystore) + // Additional sources, introduced to load remote cluster keys. + secretSources = append(secretSources, additionalSources...) // user-provided Secrets referenced in a StackConfigPolicy that configures the resource policySecretSources, err := stackconfigpolicy.GetSecureSettingsSecretSourcesForResources(ctx, r.K8sClient(), hasKeystore, hasKeystore.GetObjectKind().GroupVersionKind().Kind) if err != nil { diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index cc6cddf776..9993d0499e 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -104,6 +104,15 @@ func buildGeneralNames( {IPAddress: netutil.IPToRFCForm(netutil.LoopbackFor(netutil.ToIPFamily(podIP.String())))}, } + if cluster.Spec.RemoteClusterServer.Enabled { + // Remote cluster server is enabled. Ensure that the remote cluster service name is included in the transport certificates + // since these are the ones also used in the context of remote clusters access using API keys. + generalNames = append( + generalNames, + certificates.GeneralName{DNSName: fmt.Sprintf("%s.%s.svc", esv1.RemoteClusterService(cluster.Name), cluster.Namespace)}, + ) + } + for _, san := range cluster.Spec.Transport.TLS.SubjectAlternativeNames { if san.DNS != "" { generalNames = append(generalNames, certificates.GeneralName{DNSName: san.DNS}) diff --git a/pkg/controller/elasticsearch/client/base.go b/pkg/controller/elasticsearch/client/base.go index 6a5e2da4be..bc73fb3029 100644 --- a/pkg/controller/elasticsearch/client/base.go +++ b/pkg/controller/elasticsearch/client/base.go @@ -94,7 +94,7 @@ func (c *baseClient) get(ctx context.Context, pathWithQuery string, out interfac return c.request(ctx, http.MethodGet, pathWithQuery, nil, out, nil) } -func (c *baseClient) put(ctx context.Context, pathWithQuery string, in, out interface{}) error { //nolint:unparam +func (c *baseClient) put(ctx context.Context, pathWithQuery string, in, out interface{}) error { return c.request(ctx, http.MethodPut, pathWithQuery, in, out, nil) } @@ -106,6 +106,10 @@ func (c *baseClient) delete(ctx context.Context, pathWithQuery string) error { return c.request(ctx, http.MethodDelete, pathWithQuery, nil, nil, nil) } +func (c *baseClient) deleteWithObjects(ctx context.Context, pathWithQuery string, in, out interface{}) error { + return c.request(ctx, http.MethodDelete, pathWithQuery, in, out, nil) +} + // request performs a new http request // // if requestObj is not nil, it's marshalled as JSON and used as the request body diff --git a/pkg/controller/elasticsearch/client/client.go b/pkg/controller/elasticsearch/client/client.go index 8729ff0383..d9e3ea61e8 100644 --- a/pkg/controller/elasticsearch/client/client.go +++ b/pkg/controller/elasticsearch/client/client.go @@ -63,6 +63,7 @@ type Client interface { DesiredNodesClient ShardLister LicenseClient + RemoteClusterClient SecurityClient // Close idle connections in the underlying http client. Close() @@ -101,10 +102,6 @@ type Client interface { GetNodesStats(ctx context.Context) (NodesStats, error) // ClusterBootstrappedForZen2 returns true if the cluster is relying on zen2 orchestration. ClusterBootstrappedForZen2(ctx context.Context) (bool, error) - // UpdateRemoteClusterSettings updates the remote clusters of a cluster. - UpdateRemoteClusterSettings(ctx context.Context, settings RemoteClustersSettings) error - // GetRemoteClusterSettings retrieves the remote clusters of a cluster. - GetRemoteClusterSettings(ctx context.Context) (RemoteClustersSettings, error) // AddVotingConfigExclusions sets the transient and persistent setting of the same name in cluster settings. // Introduced in: Elasticsearch 7.0.0 AddVotingConfigExclusions(ctx context.Context, nodeNames []string) error diff --git a/pkg/controller/elasticsearch/client/model.go b/pkg/controller/elasticsearch/client/model.go index 339075ed36..062f227795 100644 --- a/pkg/controller/elasticsearch/client/model.go +++ b/pkg/controller/elasticsearch/client/model.go @@ -378,26 +378,6 @@ type StartBasicResponse struct { ErrorMessage string `json:"error_message"` } -// RemoteClustersSettings is used to build a request to update remote clusters. -type RemoteClustersSettings struct { - PersistentSettings *SettingsGroup `json:"persistent,omitempty"` -} - -// SettingsGroup is a group of persistent settings. -type SettingsGroup struct { - Cluster RemoteClusters `json:"cluster,omitempty"` -} - -// RemoteClusters models the configuration of the remote clusters. -type RemoteClusters struct { - RemoteClusters map[string]RemoteCluster `json:"remote,omitempty"` -} - -// RemoteCluster is the set of seeds to use in a remote cluster setting. -type RemoteCluster struct { - Seeds []string `json:"seeds"` -} - // Hit represents a single search hit. type Hit struct { Index string `json:"_index"` diff --git a/pkg/controller/elasticsearch/client/remote_cluster.go b/pkg/controller/elasticsearch/client/remote_cluster.go new file mode 100644 index 0000000000..e866e329b0 --- /dev/null +++ b/pkg/controller/elasticsearch/client/remote_cluster.go @@ -0,0 +1,163 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package client + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" +) + +type RemoteClusterClient interface { + // UpdateRemoteClusterSettings updates the remote clusters of a cluster. + UpdateRemoteClusterSettings(context.Context, RemoteClustersSettings) error + // GetRemoteClusterSettings retrieves the remote clusters of a cluster. + GetRemoteClusterSettings(context.Context) (RemoteClustersSettings, error) + // CreateCrossClusterAPIKey creates a new cross cluster API Key using the provided cross cluster API key request. + CreateCrossClusterAPIKey(context.Context, CrossClusterAPIKeyCreateRequest) (CrossClusterAPIKeyCreateResponse, error) + // UpdateCrossClusterAPIKey updates the cluster API Key which matches the provided ID using the provided update request. + UpdateCrossClusterAPIKey(context.Context, string, CrossClusterAPIKeyUpdateRequest) (CrossClusterAPIKeyUpdateResponse, error) + // InvalidateCrossClusterAPIKey invalidates a cluster API Key by its name. + InvalidateCrossClusterAPIKey(context.Context, string) error + // GetCrossClusterAPIKeys attempts to retrieve Cross Cluster API Keys. + // The provided string is used as the "name" parameter in the HTTP query. + // Only active API Keys are included in the response. + GetCrossClusterAPIKeys(context.Context, string) (CrossClusterAPIKeyList, error) +} + +type CrossClusterAPIKeyInvalidateRequest struct { + Name string `json:"name,omitempty"` +} + +type CrossClusterAPIKeyCreateRequest struct { + Name string `json:"name,omitempty"` + CrossClusterAPIKeyUpdateRequest +} + +type CrossClusterAPIKeyUpdateRequest struct { + esv1.RemoteClusterAPIKey + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +type CrossClusterAPIKeyCreateResponse struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + APIKey string `json:"api_key,omitempty"` + Encoded string `json:"encoded,omitempty"` +} + +type CrossClusterAPIKeyUpdateResponse struct { + Update string `json:"string,omitempty"` +} + +type CrossClusterAPIKeyList struct { + APIKeys []CrossClusterAPIKey `json:"api_keys,omitempty"` +} + +func (cl *CrossClusterAPIKeyList) Len() int { + if cl == nil { + return 0 + } + return len(cl.APIKeys) +} + +// GetElasticsearchName returns the name of the client cluster for which this key has been created. +func (c *CrossClusterAPIKey) GetElasticsearchName() (types.NamespacedName, error) { + if c == nil { + return types.NamespacedName{}, nil + } + esNameInMetadata, ok := c.Metadata["elasticsearch.k8s.elastic.co/name"] + if !ok { + return types.NamespacedName{}, fmt.Errorf("missing metadata in cross cluster API key: elasticsearch.k8s.elastic.co/name") + } + esNamespaceInMetadata, ok := c.Metadata["elasticsearch.k8s.elastic.co/namespace"] + if !ok { + return types.NamespacedName{}, fmt.Errorf("missing metadata in cross cluster API key: elasticsearch.k8s.elastic.co/namespace") + } + + namespacedName := types.NamespacedName{} + if esName, ok := esNameInMetadata.(string); ok { + namespacedName.Name = esName + } + if esNamespace, ok := esNamespaceInMetadata.(string); ok { + namespacedName.Namespace = esNamespace + } + return namespacedName, nil +} + +// GetActiveKeyWithName returns the first active key that matches the provided name or pattern. +func (cl *CrossClusterAPIKeyList) GetActiveKeyWithName(name string) *CrossClusterAPIKey { + if cl == nil || cl.Len() == 0 { + return nil + } + for _, key := range cl.APIKeys { + if key.Name == name { + return &key + } + } + return nil +} + +// KeyNames extracts the key names from a list of keys. +func (cl *CrossClusterAPIKeyList) KeyNames() sets.Set[string] { + if cl == nil || cl.Len() == 0 { + return nil + } + result := sets.New[string]() + for _, key := range cl.APIKeys { + result.Insert(key.Name) + } + return result +} + +// ForCluster returns all the API keys related to a specific client cluster. +func (cl *CrossClusterAPIKeyList) ForCluster(namespace string, name string) (*CrossClusterAPIKeyList, error) { + if cl == nil || cl.APIKeys == nil { + return nil, nil + } + crossClusterAPIKeyList := &CrossClusterAPIKeyList{ + APIKeys: make([]CrossClusterAPIKey, 0, len(cl.APIKeys)), + } + for _, apiKey := range cl.APIKeys { + elasticsearchName, err := apiKey.GetElasticsearchName() + if err != nil { + return nil, err + } + if elasticsearchName.Namespace == namespace && elasticsearchName.Name == name { + crossClusterAPIKeyList.APIKeys = append(crossClusterAPIKeyList.APIKeys, apiKey) + } + } + return crossClusterAPIKeyList, nil +} + +type CrossClusterAPIKey struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// RemoteClustersSettings is used to build a request to update remote clusters. +type RemoteClustersSettings struct { + PersistentSettings *SettingsGroup `json:"persistent,omitempty"` +} + +// SettingsGroup is a group of persistent settings. +type SettingsGroup struct { + Cluster RemoteClusters `json:"cluster,omitempty"` +} + +// RemoteClusters models the configuration of the remote clusters. +type RemoteClusters struct { + RemoteClusters map[string]RemoteCluster `json:"remote,omitempty"` +} + +// RemoteCluster is the set of seeds to use in a remote cluster setting. +type RemoteCluster struct { + Seeds []string `json:"seeds"` +} diff --git a/pkg/controller/elasticsearch/client/v6.go b/pkg/controller/elasticsearch/client/v6.go index 08efdc760e..2d361f073c 100644 --- a/pkg/controller/elasticsearch/client/v6.go +++ b/pkg/controller/elasticsearch/client/v6.go @@ -22,6 +22,22 @@ type clientV6 struct { baseClient } +func (c *clientV6) InvalidateCrossClusterAPIKey(context.Context, string) error { + return errNotSupportedInEs6x +} + +func (c *clientV6) CreateCrossClusterAPIKey(_ context.Context, _ CrossClusterAPIKeyCreateRequest) (CrossClusterAPIKeyCreateResponse, error) { + return CrossClusterAPIKeyCreateResponse{}, errNotSupportedInEs6x +} + +func (c *clientV6) UpdateCrossClusterAPIKey(_ context.Context, _ string, _ CrossClusterAPIKeyUpdateRequest) (CrossClusterAPIKeyUpdateResponse, error) { + return CrossClusterAPIKeyUpdateResponse{}, errNotSupportedInEs6x +} + +func (c *clientV6) GetCrossClusterAPIKeys(_ context.Context, _ string) (CrossClusterAPIKeyList, error) { + return CrossClusterAPIKeyList{}, errNotSupportedInEs6x +} + func (c *clientV6) Version() version.Version { return c.version } diff --git a/pkg/controller/elasticsearch/client/v8.go b/pkg/controller/elasticsearch/client/v8.go index 5f13e48353..fc52cd3537 100644 --- a/pkg/controller/elasticsearch/client/v8.go +++ b/pkg/controller/elasticsearch/client/v8.go @@ -35,6 +35,28 @@ func (c *clientV8) GetClusterState(ctx context.Context) (ClusterState, error) { return response, err } +func (c *clientV8) CreateCrossClusterAPIKey(ctx context.Context, request CrossClusterAPIKeyCreateRequest) (CrossClusterAPIKeyCreateResponse, error) { + var response CrossClusterAPIKeyCreateResponse + err := c.post(ctx, "/_security/cross_cluster/api_key", request, &response) + return response, err +} + +func (c *clientV8) UpdateCrossClusterAPIKey(ctx context.Context, apiKeyID string, request CrossClusterAPIKeyUpdateRequest) (CrossClusterAPIKeyUpdateResponse, error) { + var response CrossClusterAPIKeyUpdateResponse + err := c.put(ctx, fmt.Sprintf("/_security/cross_cluster/api_key/%s", apiKeyID), request, &response) + return response, err +} + +func (c *clientV8) GetCrossClusterAPIKeys(ctx context.Context, name string) (CrossClusterAPIKeyList, error) { + var response CrossClusterAPIKeyList + err := c.get(ctx, fmt.Sprintf("/_security/api_key?active_only=true&name=%s", name), &response) + return response, err +} + +func (c *clientV8) InvalidateCrossClusterAPIKey(ctx context.Context, name string) error { + return c.deleteWithObjects(ctx, "/_security/api_key", CrossClusterAPIKeyInvalidateRequest{Name: name}, nil) +} + // Equal returns true if c2 can be considered the same as c func (c *clientV8) Equal(c2 Client) bool { other, ok := c2.(*clientV8) diff --git a/pkg/controller/elasticsearch/driver/driver.go b/pkg/controller/elasticsearch/driver/driver.go index cae7fae24c..3a216ff362 100644 --- a/pkg/controller/elasticsearch/driver/driver.go +++ b/pkg/controller/elasticsearch/driver/driver.go @@ -11,6 +11,11 @@ import ( "strings" "time" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" @@ -149,6 +154,30 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { return results.WithError(err) } + // Remote Cluster Server (RCS2) Kubernetes Service reconciliation. + if d.ES.Spec.RemoteClusterServer.Enabled { + // Remote Cluster Server is enabled, ensure that the related Kubernetes Service does exist. + if _, err := common.ReconcileService(ctx, d.Client, services.NewRemoteClusterService(d.ES), &d.ES); err != nil { + results.WithError(err) + } + } else { + // Ensure that remote cluster Service does not exist. + remoteClusterService := &corev1.Service{} + remoteClusterServiceName := types.NamespacedName{ + Name: services.RemoteClusterServiceName(d.ES.Name), + Namespace: d.ES.Namespace, + } + if err := d.Client.Get(ctx, remoteClusterServiceName, remoteClusterService); err != nil { + if !k8serrors.IsNotFound(err) { + results.WithError(err) + } + } else { + // Remote cluster Service has been found but is not expected. + log.Info("Deleting remote cluster Service") + results.WithError(d.Client.Delete(ctx, remoteClusterService)) + } + } + resourcesState, err := reconcile.NewResourcesStateFromAPI(d.Client, d.ES) if err != nil { return results.WithError(err) @@ -325,7 +354,12 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { keystoreSecurityContext := securitycontext.For(d.Version, true) keystoreParams.SecurityContext = &keystoreSecurityContext - // setup a keystore with secure settings in an init container, if specified by the user + // setup a keystore with secure settings in an init container, if specified by the user. + // we are also using the keystore internally for the remote cluster API keys. + remoteClusterAPIKeys, err := apiKeyStoreSecretSource(ctx, &d.ES, d.Client) + if err != nil { + return results.WithError(err) + } keystoreResources, err := keystore.ReconcileResources( ctx, d, @@ -333,6 +367,7 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { esv1.ESNamer, label.NewLabels(k8s.ExtractNamespacedName(&d.ES)), keystoreParams, + remoteClusterAPIKeys..., ) if err != nil { return results.WithError(err) @@ -373,6 +408,27 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { return results.WithResults(d.reconcileNodeSpecs(ctx, esReachable, esClient, d.ReconcileState, *resourcesState, keystoreResources)) } +// apiKeyStoreSecretSource returns the Secret that holds the remote API keys, and which should be used as a secure settings source. +func apiKeyStoreSecretSource(ctx context.Context, es *esv1.Elasticsearch, c k8s.Client) ([]commonv1.NamespacedSecretSource, error) { + // Check if Secret exists + secretName := types.NamespacedName{ + Name: esv1.RemoteAPIKeysSecretName(es.Name), + Namespace: es.Namespace, + } + if err := c.Get(ctx, secretName, &corev1.Secret{}); err != nil { + if k8serrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return []commonv1.NamespacedSecretSource{ + { + Namespace: es.Namespace, + SecretName: secretName.Name, + }, + }, nil +} + // newElasticsearchClient creates a new Elasticsearch HTTP client for this cluster using the provided user func (d *defaultDriver) newElasticsearchClient( ctx context.Context, diff --git a/pkg/controller/elasticsearch/network/ports.go b/pkg/controller/elasticsearch/network/ports.go index 7a78b1dc3d..81340964b1 100644 --- a/pkg/controller/elasticsearch/network/ports.go +++ b/pkg/controller/elasticsearch/network/ports.go @@ -9,4 +9,6 @@ const ( HTTPPort = 9200 // TransportPort used by Elasticsearch for the Transport protocol in node to node communication TransportPort = 9300 + // RemoteClusterPort used by Elasticsearch for the remote cluster protocol. + RemoteClusterPort = 9443 ) diff --git a/pkg/controller/elasticsearch/nodespec/podspec_test.go b/pkg/controller/elasticsearch/nodespec/podspec_test.go index 0f4beb933d..0ca22b0556 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec_test.go +++ b/pkg/controller/elasticsearch/nodespec/podspec_test.go @@ -225,7 +225,7 @@ func TestBuildPodTemplateSpecWithDefaultSecurityContext(t *testing.T) { es.Spec.Version = tt.version.String() es.Spec.NodeSets[0].PodTemplate.Spec.SecurityContext = tt.userSecurityContext - cfg, err := settings.NewMergedESConfig(es.Name, tt.version, corev1.IPv4Protocol, es.Spec.HTTP, *es.Spec.NodeSets[0].Config, nil) + cfg, err := settings.NewMergedESConfig(es.Name, tt.version, corev1.IPv4Protocol, es.Spec.HTTP, *es.Spec.NodeSets[0].Config, nil, false, false) require.NoError(t, err) client := k8s.NewFakeClient(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: es.Namespace, Name: esv1.ScriptsConfigMap(es.Name)}}) @@ -308,7 +308,7 @@ func TestBuildPodTemplateSpec(t *testing.T) { ver, err := version.Parse(es.Spec.Version) require.NoError(t, err) - cfg, err := settings.NewMergedESConfig(es.Name, ver, corev1.IPv4Protocol, es.Spec.HTTP, *nodeSet.Config, tt.args.policyConfig.ElasticsearchConfig) + cfg, err := settings.NewMergedESConfig(es.Name, ver, corev1.IPv4Protocol, es.Spec.HTTP, *nodeSet.Config, tt.args.policyConfig.ElasticsearchConfig, false, false) require.NoError(t, err) actual, err := BuildPodTemplateSpec(context.Background(), tt.args.client, es, es.Spec.NodeSets[0], cfg, tt.args.keystoreResources, tt.args.setDefaultSecurityContext, tt.args.policyConfig) @@ -444,7 +444,7 @@ func Test_buildAnnotations(t *testing.T) { build() ver, err := version.Parse(sampleES.Spec.Version) require.NoError(t, err) - cfg, err := settings.NewMergedESConfig(es.Name, ver, corev1.IPv4Protocol, es.Spec.HTTP, *es.Spec.NodeSets[0].Config, nil) + cfg, err := settings.NewMergedESConfig(es.Name, ver, corev1.IPv4Protocol, es.Spec.HTTP, *es.Spec.NodeSets[0].Config, nil, false, false) require.NoError(t, err) got := buildAnnotations(es, cfg, tt.args.keystoreResources, tt.args.scriptsContent, tt.args.policyAnnotations) @@ -541,7 +541,7 @@ func Test_enableLog4JFormatMsgNoLookups(t *testing.T) { ver, err := version.Parse(sampleES.Spec.Version) require.NoError(t, err) - cfg, err := settings.NewMergedESConfig(sampleES.Name, ver, corev1.IPv4Protocol, sampleES.Spec.HTTP, *sampleES.Spec.NodeSets[0].Config, nil) + cfg, err := settings.NewMergedESConfig(sampleES.Name, ver, corev1.IPv4Protocol, sampleES.Spec.HTTP, *sampleES.Spec.NodeSets[0].Config, nil, false, false) require.NoError(t, err) client := k8s.NewFakeClient(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: sampleES.Namespace, Name: esv1.ScriptsConfigMap(sampleES.Name)}}) actual, err := BuildPodTemplateSpec(context.Background(), client, sampleES, sampleES.Spec.NodeSets[0], cfg, nil, false, PolicyConfig{}) diff --git a/pkg/controller/elasticsearch/nodespec/resources.go b/pkg/controller/elasticsearch/nodespec/resources.go index 155b589e76..40ffb8fced 100644 --- a/pkg/controller/elasticsearch/nodespec/resources.go +++ b/pkg/controller/elasticsearch/nodespec/resources.go @@ -81,7 +81,7 @@ func BuildExpectedResources( if nodeSpec.Config != nil { userCfg = *nodeSpec.Config } - cfg, err := settings.NewMergedESConfig(es.Name, ver, ipFamily, es.Spec.HTTP, userCfg, policyConfig.ElasticsearchConfig) + cfg, err := settings.NewMergedESConfig(es.Name, ver, ipFamily, es.Spec.HTTP, userCfg, policyConfig.ElasticsearchConfig, es.Spec.RemoteClusterServer.Enabled, es.HasRemoteClusterAPIKey()) if err != nil { return nil, err } diff --git a/pkg/controller/elasticsearch/remotecluster/elasticsearch.go b/pkg/controller/elasticsearch/remotecluster/elasticsearch.go index ffdcf8683c..7a46ba48f8 100644 --- a/pkg/controller/elasticsearch/remotecluster/elasticsearch.go +++ b/pkg/controller/elasticsearch/remotecluster/elasticsearch.go @@ -115,6 +115,11 @@ func updateSettingsInternal( remoteClustersToUpdate = append(remoteClustersToUpdate, name) // Declare remote cluster in ES seedHosts := []string{services.ExternalTransportServiceHost(remoteCluster.ElasticsearchRef.NamespacedName())} + if remoteCluster.APIKey != nil { + // User specified an API key. It means that the remote cluster is expected to be accessed using the remote + // cluster Service instead of relying on the transport layer. + seedHosts = []string{services.RemoteClusterServerServiceHost(remoteCluster.ElasticsearchRef.NamespacedName())} + } remoteClustersToApply[name] = esclient.RemoteCluster{Seeds: seedHosts} // Ensure this cluster is tracked in the annotation remoteClustersInAnnotation[name] = struct{}{} diff --git a/pkg/controller/elasticsearch/services/services.go b/pkg/controller/elasticsearch/services/services.go index 71284a22bb..b2b23d18a7 100644 --- a/pkg/controller/elasticsearch/services/services.go +++ b/pkg/controller/elasticsearch/services/services.go @@ -5,7 +5,6 @@ package services import ( - "context" "fmt" "math/rand" "strconv" @@ -75,11 +74,22 @@ func InternalServiceName(esName string) string { return esv1.InternalHTTPService(esName) } +// RemoteClusterServiceName returns the name for the remote cluster service used when the cluster is expected to be accessed +// using the remote cluster server. Managed by the operator exclusively. +func RemoteClusterServiceName(esName string) string { + return esv1.RemoteClusterService(esName) +} + // ExternalTransportServiceHost returns the hostname and the port used to reach Elasticsearch's transport endpoint. func ExternalTransportServiceHost(es types.NamespacedName) string { return stringsutil.Concat(TransportServiceName(es.Name), ".", es.Namespace, globalServiceSuffix, ":", strconv.Itoa(network.TransportPort)) } +// RemoteClusterServerServiceHost returns the hostname and the port used to reach Elasticsearch's remote cluster server endpoint. +func RemoteClusterServerServiceHost(es types.NamespacedName) string { + return stringsutil.Concat(RemoteClusterServiceName(es.Name), ".", es.Namespace, globalServiceSuffix, ":", strconv.Itoa(network.RemoteClusterPort)) +} + // ExternalServiceURL returns the URL used to reach Elasticsearch's external endpoint. func ExternalServiceURL(es esv1.Elasticsearch) string { return stringsutil.Concat(es.Spec.HTTP.Protocol(), "://", ExternalServiceName(es.Name), ".", es.Namespace, globalServiceSuffix, ":", strconv.Itoa(network.HTTPPort)) @@ -141,45 +151,27 @@ func NewInternalService(es esv1.Elasticsearch) *corev1.Service { } } -// IsServiceReady checks if a service has one or more ready endpoints. -func IsServiceReady(c k8s.Client, service corev1.Service) (bool, error) { - endpoints := corev1.Endpoints{} - namespacedName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} - - if err := c.Get(context.Background(), namespacedName, &endpoints); err != nil { - return false, err - } - for _, subs := range endpoints.Subsets { - if len(subs.Addresses) > 0 { - return true, nil - } - } - return false, nil -} - -// GetExternalService returns the external service associated to the given Elasticsearch cluster. -func GetExternalService(c k8s.Client, es esv1.Elasticsearch) (corev1.Service, error) { - return getServiceByName(c, es, ExternalServiceName(es.Name)) -} - -// GetInternalService returns the internally managed service associated to the given Elasticsearch cluster. -func GetInternalService(c k8s.Client, es esv1.Elasticsearch) (corev1.Service, error) { - return getServiceByName(c, es, InternalServiceName(es.Name)) -} - -func getServiceByName(c k8s.Client, es esv1.Elasticsearch, serviceName string) (corev1.Service, error) { - var svc corev1.Service - - namespacedName := types.NamespacedName{ - Namespace: es.Namespace, - Name: serviceName, - } - - if err := c.Get(context.Background(), namespacedName, &svc); err != nil { - return corev1.Service{}, err +// NewRemoteClusterService returns the service associated to the remote cluster service for the given cluster. +func NewRemoteClusterService(es esv1.Elasticsearch) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: RemoteClusterServiceName(es.Name), + Namespace: es.Namespace, + Labels: label.NewLabels(k8s.ExtractNamespacedName(&es)), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{ + { + Name: "rcs", + Protocol: corev1.ProtocolTCP, + Port: network.RemoteClusterPort, + }, + }, + Selector: label.NewLabels(k8s.ExtractNamespacedName(&es)), + PublishNotReadyAddresses: false, + }, } - - return svc, nil } type urlProvider struct { diff --git a/pkg/controller/elasticsearch/settings/merged_config.go b/pkg/controller/elasticsearch/settings/merged_config.go index d02e12a103..5e6dbfe682 100644 --- a/pkg/controller/elasticsearch/settings/merged_config.go +++ b/pkg/controller/elasticsearch/settings/merged_config.go @@ -34,15 +34,16 @@ func NewMergedESConfig( httpConfig commonv1.HTTPConfig, userConfig commonv1.Config, esConfigFromStackConfigPolicy *common.CanonicalConfig, + remoteClusterServerEnabled, remoteClusterClientEnabled bool, ) (CanonicalConfig, error) { userCfg, err := common.NewCanonicalConfigFrom(userConfig.Data) if err != nil { return CanonicalConfig{}, err } - config := baseConfig(clusterName, ver, ipFamily).CanonicalConfig + config := baseConfig(clusterName, ver, ipFamily, remoteClusterServerEnabled).CanonicalConfig err = config.MergeWith( - xpackConfig(ver, httpConfig).CanonicalConfig, + xpackConfig(ver, httpConfig, remoteClusterServerEnabled, remoteClusterClientEnabled).CanonicalConfig, userCfg, esConfigFromStackConfigPolicy, ) @@ -53,7 +54,7 @@ func NewMergedESConfig( } // baseConfig returns the base ES configuration to apply for the given cluster -func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamily) *CanonicalConfig { +func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamily, remoteClusterServerEnabled bool) *CanonicalConfig { cfg := map[string]interface{}{ // derive node name dynamically from the pod name, injected as env var esv1.NodeName: "${" + EnvPodName + "}", @@ -72,6 +73,10 @@ func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamil esv1.PathLogs: volume.ElasticsearchLogsMountPath, } + if remoteClusterServerEnabled { + cfg[esv1.RemoteClusterEnabled] = "true" + } + // seed hosts setting name changed starting ES 7.X fileProvider := "file" if ver.Major < 7 { @@ -91,7 +96,7 @@ func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamil } // xpackConfig returns the configuration bit related to XPack settings -func xpackConfig(ver version.Version, httpCfg commonv1.HTTPConfig) *CanonicalConfig { +func xpackConfig(ver version.Version, httpCfg commonv1.HTTPConfig, remoteClusterServerEnabled, remoteClusterClientEnabled bool) *CanonicalConfig { // enable x-pack security, including TLS cfg := map[string]interface{}{ // x-pack security general settings @@ -121,6 +126,26 @@ func xpackConfig(ver version.Version, httpCfg commonv1.HTTPConfig) *CanonicalCon esv1.XPackSecurityHttpSslCertificateAuthorities: path.Join(volume.HTTPCertificatesSecretVolumeMountPath, certificates.CAFileName), } + if remoteClusterServerEnabled { + cfg[esv1.XPackSecurityRemoteClusterServerSslKey] = path.Join( + volume.TransportCertificatesSecretVolumeMountPath, + "${POD_NAME}."+certificates.KeyFileName, + ) + cfg[esv1.XPackSecurityRemoteClusterServerSslCertificate] = path.Join( + volume.TransportCertificatesSecretVolumeMountPath, + "${POD_NAME}."+certificates.CertFileName, + ) + cfg[esv1.XPackSecurityRemoteClusterServerSslCertificateAuthorities] = []string{ + path.Join(volume.TransportCertificatesSecretVolumeMountPath, certificates.CAFileName), + path.Join(volume.RemoteCertificateAuthoritiesSecretVolumeMountPath, certificates.CAFileName), + } + } + + if remoteClusterClientEnabled { + cfg[esv1.XPackSecurityRemoteClusterClientSslKey] = true + cfg[esv1.XPackSecurityRemoteClusterClientSslCertificateAuthorities] = path.Join(volume.RemoteCertificateAuthoritiesSecretVolumeMountPath, certificates.CAFileName) + } + // always enable the built-in file and native internal realms for user auth, ordered as first if ver.Major < 7 { // 6.x syntax diff --git a/pkg/controller/elasticsearch/settings/merged_config_test.go b/pkg/controller/elasticsearch/settings/merged_config_test.go index 848d504074..e932c961cc 100644 --- a/pkg/controller/elasticsearch/settings/merged_config_test.go +++ b/pkg/controller/elasticsearch/settings/merged_config_test.go @@ -42,13 +42,77 @@ func TestNewMergedESConfig(t *testing.T) { }) tests := []struct { - name string - version string - ipFamily corev1.IPFamily - cfgData map[string]interface{} - policyCfgData *common.CanonicalConfig - assert func(cfg CanonicalConfig) + name string + version string + ipFamily corev1.IPFamily + remoteClusterServerEnabled bool + remoteClusterClientEnabled bool + cfgData map[string]interface{} + policyCfgData *common.CanonicalConfig + assert func(cfg CanonicalConfig) }{ + { + name: "No remote cluster client or server by default", + version: "8.15.0", + ipFamily: corev1.IPv4Protocol, + cfgData: map[string]interface{}{}, + assert: func(cfg CanonicalConfig) { + // Remote cluster client configuration. + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + // Remote cluster server configuration. + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.key"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate_authorities"}))) + }, + }, + { + name: "Remote cluster client is enabled", + version: "8.15.0", + ipFamily: corev1.IPv4Protocol, + remoteClusterClientEnabled: true, + cfgData: map[string]interface{}{}, + assert: func(cfg CanonicalConfig) { + // Remote cluster client configuration. + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + // Remote cluster server configuration. + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.key"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate_authorities"}))) + }, + }, + { + name: "Remote cluster server is enabled", + version: "8.15.0", + ipFamily: corev1.IPv4Protocol, + remoteClusterServerEnabled: true, + cfgData: map[string]interface{}{}, + assert: func(cfg CanonicalConfig) { + // Remote cluster client configuration. + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + // Remote cluster server configuration. + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.key"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate_authorities"}))) + }, + }, + { + name: "in 6.x, empty config should have the default file and native realm settings configured", + version: "6.8.0", + ipFamily: corev1.IPv4Protocol, + cfgData: map[string]interface{}{}, + assert: func(cfg CanonicalConfig) { + require.Equal(t, 0, len(cfg.HasKeys([]string{nodeML}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.XPackSecurityAuthcRealmsFile1Type}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.XPackSecurityAuthcRealmsFile1Order}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.XPackSecurityAuthcRealmsNative1Type}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.XPackSecurityAuthcRealmsNative1Order}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.ShardAwarenessAttributes}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{nodeAttrNodeName}))) + }, + }, { name: "in 6.x, empty config should have the default file and native realm settings configured", version: "6.8.0", @@ -259,7 +323,7 @@ func TestNewMergedESConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ver, err := version.Parse(tt.version) require.NoError(t, err) - cfg, err := NewMergedESConfig("clusterName", ver, tt.ipFamily, commonv1.HTTPConfig{}, commonv1.Config{Data: tt.cfgData}, tt.policyCfgData) + cfg, err := NewMergedESConfig("clusterName", ver, tt.ipFamily, commonv1.HTTPConfig{}, commonv1.Config{Data: tt.cfgData}, tt.policyCfgData, tt.remoteClusterServerEnabled, tt.remoteClusterClientEnabled) require.NoError(t, err) tt.assert(cfg) }) diff --git a/pkg/controller/elasticsearch/validation/validations.go b/pkg/controller/elasticsearch/validation/validations.go index d53ea67304..04a20e02ef 100644 --- a/pkg/controller/elasticsearch/validation/validations.go +++ b/pkg/controller/elasticsearch/validation/validations.go @@ -73,6 +73,7 @@ func validations(ctx context.Context, checker license.Checker, exposedNodeLabels validPVCNaming, validMonitoring, validAssociations, + supportsRemoteClusterUsingAPIKey, func(proposed esv1.Elasticsearch) field.ErrorList { return validLicenseLevel(ctx, proposed, checker) }, @@ -134,6 +135,37 @@ func supportedVersion(es esv1.Elasticsearch) field.ErrorList { return field.ErrorList{field.Invalid(field.NewPath("spec").Child("version"), es.Spec.Version, unsupportedVersionMsg)} } +func supportsRemoteClusterUsingAPIKey(es esv1.Elasticsearch) field.ErrorList { + ver, err := version.Parse(es.Spec.Version) + if err != nil { + return field.ErrorList{field.Invalid(field.NewPath("spec").Child("version"), es.Spec.Version, parseVersionErrMsg)} + } + var errs field.ErrorList + if es.Spec.RemoteClusterServer.Enabled && ver.LE(esv1.RemoteClusterAPIKeysMinVersion) { + errs = append(errs, field.Invalid( + field.NewPath("spec").Child("remoteClusterServer"), + es.Spec.Version, + fmt.Sprintf( + "minimum required version for remote cluster server is %s but desired version is %s", + esv1.RemoteClusterAPIKeysMinVersion, + es.Spec.Version, + ), + )) + } + if es.HasRemoteClusterAPIKey() && ver.LE(esv1.RemoteClusterAPIKeysMinVersion) { + errs = append(errs, field.Invalid( + field.NewPath("spec").Child("remoteClusters").Child("*").Key("apiKey"), + es.Spec.Version, + fmt.Sprintf( + "minimum required version for remote cluster using API keys is %s but desired version is %s", + esv1.RemoteClusterAPIKeysMinVersion, + es.Spec.Version, + ), + )) + } + return errs +} + // hasCorrectNodeRoles checks whether Elasticsearch node roles are correctly configured. // The rules are: // There must be at least one master node. diff --git a/pkg/controller/elasticsearch/validation/validations_test.go b/pkg/controller/elasticsearch/validation/validations_test.go index ea8d7935df..0e1ce64376 100644 --- a/pkg/controller/elasticsearch/validation/validations_test.go +++ b/pkg/controller/elasticsearch/validation/validations_test.go @@ -203,6 +203,104 @@ func Test_supportedVersion(t *testing.T) { } } +func Test_supportsRemoteClusterUsingAPIKey(t *testing.T) { + tests := []struct { + name string + es esv1.Elasticsearch + expectErrors bool + }{ + { + name: "no remote cluster settings that relies on API keys", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + }, + Spec: esv1.ElasticsearchSpec{Version: "7.0.0"}, + }, + expectErrors: false, + }, + { + name: "some remote cluster API keys before required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.9.99", + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "bar", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + }, + }, + }, + expectErrors: true, + }, + { + name: "some remote cluster API keys with min required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.10.0", + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "bar", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + }, + }, + }, + expectErrors: false, + }, + { + name: "remote cluster without API keys before required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.9.99", + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "bar", + }, + }, + }, + }, + expectErrors: false, + }, + { + name: "remote cluster server enabled with min required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.10.0", + RemoteClusterServer: esv1.RemoteClusterServer{Enabled: true}, + }, + }, + expectErrors: false, + }, + { + name: "remote cluster server enabled before min required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.9.99", + RemoteClusterServer: esv1.RemoteClusterServer{Enabled: true}, + }, + }, + expectErrors: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := supportsRemoteClusterUsingAPIKey(tt.es) + actualErrors := len(actual) > 0 + if tt.expectErrors != actualErrors { + t.Errorf("failed supportsRemoteClusterUsingAPIKey(). Name: %v, actual %v, wanted: %v, value: %v", tt.name, actual, tt.expectErrors, tt.es.Spec.Version) + } + }) + } +} + func Test_validName(t *testing.T) { tests := []struct { name string diff --git a/pkg/controller/remoteca/controller_test.go b/pkg/controller/remoteca/controller_test.go deleted file mode 100644 index 7e00af14be..0000000000 --- a/pkg/controller/remoteca/controller_test.go +++ /dev/null @@ -1,456 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package remoteca - -import ( - "context" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" - esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/license" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/watches" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/certificates/transport" - "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" - "github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac" -) - -type clusterBuilder struct { - name, namespace string - remoteClusters []commonv1.ObjectSelector -} - -func newClusteBuilder(namespace, name string) *clusterBuilder { - return &clusterBuilder{ - name: name, - namespace: namespace, - } -} - -func (cb *clusterBuilder) withRemoteCluster(namespace, name string) *clusterBuilder { - cb.remoteClusters = append(cb.remoteClusters, commonv1.ObjectSelector{ - Name: name, - Namespace: namespace, - }) - return cb -} - -func (cb *clusterBuilder) build() *esv1.Elasticsearch { - remoteClusters := make([]esv1.RemoteCluster, len(cb.remoteClusters)) - for i, remoteCluster := range cb.remoteClusters { - remoteClusters[i] = esv1.RemoteCluster{ - ElasticsearchRef: commonv1.LocalObjectSelector{ - Name: remoteCluster.Name, - Namespace: remoteCluster.Namespace, - }} - } - - return &esv1.Elasticsearch{ - ObjectMeta: v1.ObjectMeta{ - Namespace: cb.namespace, - Name: cb.name, - }, - Spec: esv1.ElasticsearchSpec{ - RemoteClusters: remoteClusters, - }, - } -} - -type fakeAccessReviewer struct { - allowed bool - err error -} - -func (f *fakeAccessReviewer) AccessAllowed(_ context.Context, _ string, _ string, _ runtime.Object) (bool, error) { - return f.allowed, f.err -} - -func fakePublicCa(namespace, name string) *corev1.Secret { - namespacedName := types.NamespacedName{ - Name: name, - Namespace: namespace, - } - transportPublicCertKey := transport.PublicCertsSecretRef(namespacedName) - return &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Namespace: transportPublicCertKey.Namespace, - Name: transportPublicCertKey.Name, - }, - Data: map[string][]byte{ - certificates.CAFileName: []byte(namespacedName.String()), - }, - } -} - -// remoteCa builds an expected remote Ca -func remoteCa(localNamespace, localName, remoteNamespace, remoteName string) *corev1.Secret { - remoteNamespacedName := types.NamespacedName{ - Name: remoteName, - Namespace: remoteNamespace, - } - return &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Namespace: localNamespace, - Name: remoteCASecretName(localName, remoteNamespacedName), - Labels: map[string]string{ - "common.k8s.elastic.co/type": "remote-ca", - "elasticsearch.k8s.elastic.co/cluster-name": localName, - "elasticsearch.k8s.elastic.co/remote-cluster-name": remoteName, - "elasticsearch.k8s.elastic.co/remote-cluster-namespace": remoteNamespace, - }, - }, - Data: map[string][]byte{ - certificates.CAFileName: []byte(remoteNamespacedName.String()), - }, - } -} - -func withDataCert(caSecret *corev1.Secret, newCa []byte) *corev1.Secret { - caSecret.Data[certificates.CAFileName] = newCa - return caSecret -} - -func TestReconcileRemoteCa_Reconcile(t *testing.T) { - type fields struct { - clusters []client.Object - accessReviewer rbac.AccessReviewer - licenseChecker license.Checker - } - type args struct { - request reconcile.Request - } - tests := []struct { - name string - fields fields - args args - - expectedSecrets []*corev1.Secret - unexpectedSecrets []types.NamespacedName - want reconcile.Result - wantErr bool - }{ - { - name: "Simple remote cluster ns1/es1 -> ns2/es2", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - expectedSecrets: []*corev1.Secret{ - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "Bi-directional remote cluster ns1/es1 <-> ns2/es2", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").withRemoteCluster("ns1", "es1").build(), - fakePublicCa("ns2", "es2"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - expectedSecrets: []*corev1.Secret{ - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "Deleted remote cluster", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - unexpectedSecrets: []types.NamespacedName{ - { - Namespace: "ns1", - Name: remoteCASecretName("es1", types.NamespacedName{ - Namespace: "ns2", - Name: "es2", - }), - }, - { - Namespace: "ns2", - Name: remoteCASecretName("es2", types.NamespacedName{ - Namespace: "ns1", - Name: "es1", - }), - }, - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "CA content has been updated, remote ca must be reconciled", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - withDataCert(remoteCa("ns1", "es1", "ns2", "es2"), []byte("foo")), - withDataCert(remoteCa("ns2", "es2", "ns1", "es1"), []byte("bar")), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - expectedSecrets: []*corev1.Secret{ - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - // ns1/es1 has been deleted - all related secrets in other namespaces must be deleted - name: "Deleted cluster", - fields: fields{ - clusters: []client.Object{ - // ns2/es2 - newClusteBuilder("ns2", "es2").withRemoteCluster("ns1", "es1").build(), - fakePublicCa("ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - // ns3/es3 - newClusteBuilder("ns3", "es3").withRemoteCluster("ns1", "es1").build(), - fakePublicCa("ns3", "es3"), - remoteCa("ns3", "es3", "ns1", "es1"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - unexpectedSecrets: []types.NamespacedName{ - { - Namespace: "ns3", - Name: remoteCASecretName("es3", types.NamespacedName{ - Namespace: "ns1", - Name: "es1", - }), - }, - { - Namespace: "ns2", - Name: remoteCASecretName("es2", types.NamespacedName{ - Namespace: "ns1", - Name: "es1", - }), - }, - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "No enterprise license, remote ca are not created", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: false}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - unexpectedSecrets: []types.NamespacedName{ - { - Namespace: "ns1", - Name: remoteCASecretName("es1", types.NamespacedName{ - Namespace: "ns2", - Name: "es2", - }), - }, - { - Namespace: "ns2", - Name: remoteCASecretName("es2", types.NamespacedName{ - Namespace: "ns1", - Name: "es1", - }), - }, - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "No enterprise license, existing remote ca are left untouched", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: false}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - expectedSecrets: []*corev1.Secret{ - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "Association is not allowed, existing remote ca are removed", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - accessReviewer: &fakeAccessReviewer{allowed: false}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - unexpectedSecrets: []types.NamespacedName{ - { - Namespace: "ns1", - Name: remoteCASecretName("es1", types.NamespacedName{Namespace: "ns2", Name: "es2"}), - }, - { - Namespace: "ns2", - Name: remoteCASecretName("es2", types.NamespacedName{Namespace: "ns1", Name: "es1"}), - }, - }, - want: reconcile.Result{}, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - w := watches.NewDynamicWatches() - r := &ReconcileRemoteCa{ - Client: k8s.NewFakeClient(tt.fields.clusters...), - accessReviewer: tt.fields.accessReviewer, - watches: w, - licenseChecker: tt.fields.licenseChecker, - recorder: record.NewFakeRecorder(10), - } - got, err := r.Reconcile(context.Background(), tt.args.request) - if (err != nil) != tt.wantErr { - t.Errorf("ReconcileRemoteCa.Reconcile() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ReconcileRemoteCa.Reconcile() = %v, want %v", got, tt.want) - } - // Check that expected secrets are here - for _, expectedSecret := range tt.expectedSecrets { - var actualSecret corev1.Secret - assert.NoError(t, r.Client.Get(context.Background(), types.NamespacedName{Namespace: expectedSecret.Namespace, Name: expectedSecret.Name}, &actualSecret)) - // Compare content - actualCa, ok := actualSecret.Data[certificates.CAFileName] - assert.True(t, ok) - assert.Equal(t, expectedSecret.Data[certificates.CAFileName], actualCa) - // Compare labels - assert.NotNil(t, actualSecret.Labels) - assert.Equal(t, expectedSecret.Labels, actualSecret.Labels) - } - // Check that unexpected secrets does not exist - for _, unexpectedSecret := range tt.unexpectedSecrets { - var actualSecret corev1.Secret - err := r.Client.Get(context.Background(), types.NamespacedName{Namespace: unexpectedSecret.Namespace, Name: unexpectedSecret.Name}, &actualSecret) - assert.True(t, apierrors.IsNotFound(err)) - } - }) - } -} diff --git a/pkg/controller/remotecluster/apikey.go b/pkg/controller/remotecluster/apikey.go new file mode 100644 index 0000000000..b27e82e6dd --- /dev/null +++ b/pkg/controller/remotecluster/apikey.go @@ -0,0 +1,156 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/util/sets" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/hash" + esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" +) + +// reconcileAPIKeys creates or updates the API Keys for the remote/client cluster, +// which may have several references (.Spec.RemoteClusters) to the cluster being reconciled. +func reconcileAPIKeys( + ctx context.Context, + c k8s.Client, + activeAPIKeys esclient.CrossClusterAPIKeyList, // all the API Keys in the reconciled/local cluster + reconciledES *esv1.Elasticsearch, // the Elasticsearch cluster being reconciled, where the API keys must be created/invalidated + clientES *esv1.Elasticsearch, // the remote Elasticsearch cluster which is going to act as the client, where the API keys are going to be store in the keystore Secret + remoteClusters []esv1.RemoteCluster, // the expected API keys for that client cluster + esClient esclient.Client, // ES client for the remote cluster which is going to act as the client +) error { + // clientClusterAPIKeyStore is used to reconcile encoded API keys in the client cluster, to inject new API keys + // or to delete the ones which are no longer needed. + clientClusterAPIKeyStore, err := LoadAPIKeyStore(ctx, c, clientES) + if err != nil { + return err + } + + log := ulog.FromContext(ctx).WithValues( + "local_namespace", reconciledES.Namespace, + "local_name", reconciledES.Name, + "remote_namespace", clientES.Namespace, + "remote_name", clientES.Name, + ) + + // Maintain a list of the expected API keys for that specific client cluster, to detect the ones which are no longer expected in the reconciled cluster. + expectedKeysInReconciledES := sets.New[string]() + // Same for the aliases + expectedAliases := sets.New[string]() + activeAPIKeysNames := activeAPIKeys.KeyNames() + for _, remoteCluster := range remoteClusters { + apiKeyName := fmt.Sprintf("eck-%s-%s-%s", clientES.Namespace, clientES.Name, remoteCluster.Name) + expectedKeysInReconciledES.Insert(apiKeyName) + expectedAliases.Insert(remoteCluster.Name) + if remoteCluster.APIKey == nil { + if activeAPIKeysNames.Has(apiKeyName) { + // We found an API key for that client cluster while it is not expected to have one. + // It may happen when the user switched back from API keys to the legacy remote cluster. + log.Info("Invalidating API key as remote cluster is not configured to use it", "alias", remoteCluster.Name) + if err := esClient.InvalidateCrossClusterAPIKey(ctx, apiKeyName); err != nil { + return err + } + } + continue + } + + // Attempt to get an existing API Key with that key name. + activeAPIKey := activeAPIKeys.GetActiveKeyWithName(apiKeyName) + expectedHash := hash.HashObject(remoteCluster.APIKey) + if activeAPIKey == nil { + // Active API key not found, let's create a new one. + log.Info("Creating API key", "alias", remoteCluster.Name, "key", apiKeyName) + apiKey, err := esClient.CreateCrossClusterAPIKey(ctx, esclient.CrossClusterAPIKeyCreateRequest{ + Name: apiKeyName, + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + RemoteClusterAPIKey: *remoteCluster.APIKey, + Metadata: newMetadataFor(clientES, expectedHash), + }, + }) + if err != nil { + return err + } + clientClusterAPIKeyStore.Update(reconciledES.Name, reconciledES.Namespace, remoteCluster.Name, apiKey.ID, apiKey.Encoded) + } + // If an API key already exists ensure that the access field is the expected one using the hash + if activeAPIKey != nil { + // Ensure that the API key is in the keystore + if clientClusterAPIKeyStore.KeyIDFor(remoteCluster.Name) != activeAPIKey.ID { + // We have a problem here, the API Key ID in Elasticsearch does not match the API Key recorded in the Secret. + // Invalidate the API Key in ES and requeue + log.Info("Invalidating API key as it does not match the one in keystore", "alias", remoteCluster.Name, "key", apiKeyName) + if err := esClient.InvalidateCrossClusterAPIKey(ctx, activeAPIKey.Name); err != nil { + return err + } + return fmt.Errorf( + "cluster key id for alias %s %s (%s), does not match the one stored in the keystore of %s/%s", + remoteCluster.Name, activeAPIKey.Name, activeAPIKey.ID, clientES.Namespace, clientES.Name, + ) + } + currentHash := activeAPIKey.Metadata["elasticsearch.k8s.elastic.co/config-hash"] + if currentHash != expectedHash { + log.Info("Updating API key", "alias", remoteCluster.Name) + // Update the Key + _, err := esClient.UpdateCrossClusterAPIKey(ctx, activeAPIKey.ID, esclient.CrossClusterAPIKeyUpdateRequest{ + RemoteClusterAPIKey: *remoteCluster.APIKey, + Metadata: newMetadataFor(clientES, expectedHash), + }) + if err != nil { + return err + } + } + } + } + + // Get all the active API keys which have been created for that client cluster. + activeAPIKeysForClientCluster, err := activeAPIKeys.ForCluster(clientES.Namespace, clientES.Name) + if err != nil { + return err + } + // Invalidate all the keys related to that local cluster which are not expected. + for keyName := range activeAPIKeysForClientCluster.KeyNames() { + if !expectedKeysInReconciledES.Has(keyName) { + // Unexpected key, let's invalidate it. + log.Info("Invalidating unexpected API key", "key", keyName) + if err := esClient.InvalidateCrossClusterAPIKey(ctx, keyName); err != nil { + return err + } + } + } + + // Delete all the keys in the keystore which are not expected. + aliases := clientClusterAPIKeyStore.ForCluster(reconciledES.Namespace, reconciledES.Name) + for existingAlias := range aliases { + if expectedAliases.Has(existingAlias) { + continue + } + clientClusterAPIKeyStore.Delete(existingAlias) + } + + // Save the generated keys in the keystore. + if err := clientClusterAPIKeyStore.Save(ctx, c, clientES); err != nil { + return err + } + return nil +} + +// newMetadataFor returns the metadata to be set in the Elasticsearch API keys metadata in the Elasticsearch cluster +// state, not on a Kubernetes object. +func newMetadataFor(clientES *esv1.Elasticsearch, expectedHash string) map[string]interface{} { + return map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": expectedHash, + "elasticsearch.k8s.elastic.co/name": clientES.Name, + "elasticsearch.k8s.elastic.co/namespace": clientES.Namespace, + "elasticsearch.k8s.elastic.co/uid": clientES.UID, + "elasticsearch.k8s.elastic.co/managed-by": "eck", + } +} diff --git a/pkg/controller/remoteca/controller.go b/pkg/controller/remotecluster/controller.go similarity index 50% rename from pkg/controller/remoteca/controller.go rename to pkg/controller/remotecluster/controller.go index b60337b06d..0d14a9d1b8 100644 --- a/pkg/controller/remoteca/controller.go +++ b/pkg/controller/remotecluster/controller.go @@ -2,13 +2,18 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "context" "fmt" "time" + "k8s.io/apimachinery/pkg/util/sets" + + commonesclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/esclient" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services" + "go.elastic.co/apm/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -28,6 +33,7 @@ import ( "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/tracing" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/watches" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/certificates/remoteca" + esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" @@ -35,16 +41,16 @@ import ( ) const ( - name = "remoteca-controller" + name = "remotecluster-controller" EventReasonClusterCaCertNotFound = "ClusterCaCertNotFound" ) var ( - defaultRequeue = reconcile.Result{Requeue: true, RequeueAfter: 20 * time.Second} + defaultRequeue = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} ) -// Add creates a new RemoteCa Controller and adds it to the manager with default RBAC. +// Add creates a new ReconcileRemoteClusters Controller and adds it to the manager with default RBAC. func Add(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operator.Parameters) error { r := NewReconciler(mgr, accessReviewer, params) c, err := common.NewController(mgr, name, r, params) @@ -55,28 +61,30 @@ func Add(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operato } // NewReconciler returns a new reconcile.Reconciler -func NewReconciler(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operator.Parameters) *ReconcileRemoteCa { +func NewReconciler(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operator.Parameters) *ReconcileRemoteClusters { c := mgr.GetClient() - return &ReconcileRemoteCa{ - Client: c, - accessReviewer: accessReviewer, - watches: watches.NewDynamicWatches(), - recorder: mgr.GetEventRecorderFor(name), - licenseChecker: license.NewLicenseChecker(c, params.OperatorNamespace), - Parameters: params, + return &ReconcileRemoteClusters{ + Client: c, + accessReviewer: accessReviewer, + watches: watches.NewDynamicWatches(), + recorder: mgr.GetEventRecorderFor(name), + licenseChecker: license.NewLicenseChecker(c, params.OperatorNamespace), + Parameters: params, + esClientProvider: commonesclient.NewClient, } } -var _ reconcile.Reconciler = &ReconcileRemoteCa{} +var _ reconcile.Reconciler = &ReconcileRemoteClusters{} -// ReconcileRemoteCa reconciles remote CA Secrets. -type ReconcileRemoteCa struct { +// ReconcileRemoteClusters reconciles remote clusters Secrets and API Keys. +type ReconcileRemoteClusters struct { k8s.Client operator.Parameters - accessReviewer rbac.AccessReviewer - recorder record.EventRecorder - watches watches.DynamicWatches - licenseChecker license.Checker + accessReviewer rbac.AccessReviewer + recorder record.EventRecorder + watches watches.DynamicWatches + licenseChecker license.Checker + esClientProvider commonesclient.Provider // iteration is the number of times this controller has run its Reconcile method iteration uint64 @@ -84,7 +92,7 @@ type ReconcileRemoteCa struct { // Reconcile reads that state of the cluster for the expected remote clusters in this Kubernetes cluster. // It copies the remote CA Secrets so they can be trusted by every peer Elasticsearch clusters. -func (r *ReconcileRemoteCa) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { +func (r *ReconcileRemoteClusters) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { ctx = common.NewReconciliationContext(ctx, &r.iteration, r.Tracer, name, "es_name", request) defer common.LogReconciliationRun(ulog.FromContext(ctx))() defer tracing.EndContextTransaction(ctx) @@ -103,21 +111,20 @@ func (r *ReconcileRemoteCa) Reconcile(ctx context.Context, request reconcile.Req ulog.FromContext(ctx).Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", es.Namespace, "es_name", es.Name) return reconcile.Result{}, nil } - return doReconcile(ctx, r, &es) } // deleteAllRemoteCa deletes all associated remote certificate authorities -func deleteAllRemoteCa(ctx context.Context, r *ReconcileRemoteCa, es types.NamespacedName) (reconcile.Result, error) { +func deleteAllRemoteCa(ctx context.Context, r *ReconcileRemoteClusters, es types.NamespacedName) (reconcile.Result, error) { span, _ := apm.StartSpan(ctx, "delete_all_remote_ca", tracing.SpanTypeApp) defer span.End() - remoteClusters, err := remoteClustersInvolvedWith(ctx, r.Client, es) + associatedCAs, err := getAssociatedRemoteCAs(ctx, r.Client, es) if err != nil { return reconcile.Result{}, err } results := &reconciler.Results{} - for remoteCluster := range remoteClusters { + for remoteCluster := range associatedCAs { if err := deleteCertificateAuthorities(ctx, r, es, remoteCluster); err != nil { results.WithError(err) } @@ -127,7 +134,7 @@ func deleteAllRemoteCa(ctx context.Context, r *ReconcileRemoteCa, es types.Names func doReconcile( ctx context.Context, - r *ReconcileRemoteCa, + r *ReconcileRemoteClusters, localEs *esv1.Elasticsearch, ) (reconcile.Result, error) { log := ulog.FromContext(ctx) @@ -152,42 +159,156 @@ func doReconcile( } // Get all the clusters to which this reconciled cluster is connected to according to the existing remote CAs. - // remoteClustersInvolved is used to delete the CA certificates and cancel any trust relationships + // associatedRemoteCAs is used to delete the CA certificates and cancel any trust relationships // that may have existed in the past but should not exist anymore. - remoteClustersInvolved, err := remoteClustersInvolvedWith(ctx, r.Client, localClusterKey) + associatedRemoteCAs, err := getAssociatedRemoteCAs(ctx, r.Client, localClusterKey) if err != nil { return reconcile.Result{}, err } + var ( + activeAPIKeys esclient.CrossClusterAPIKeyList + esClient esclient.Client + ) + localClusterSupportClusterAPIKeys, err := localEs.SupportRemoteClusterAPIKeys() + if err != nil { + return reconcile.Result{}, err + } results := &reconciler.Results{} - // Create or update expected remote CA - for remoteEsKey := range expectedRemoteClusters { - // Get the remote Elasticsearch cluster associated with this remote CA + if localClusterSupportClusterAPIKeys.IsTrue() { + // Check if the ES API is available. We need it to create, update and invalidate + // API keys in this cluster. + if !services.NewElasticsearchURLProvider(*localEs, r.Client).HasEndpoints() { + log.Info("Elasticsearch API is not available yet") + return results.WithResult(defaultRequeue).Aggregate() + } + // Create a new client + newEsClient, err := r.esClientProvider(ctx, r.Client, r.Dialer, *localEs) + if err != nil { + return reconcile.Result{}, err + } + // Check that the API is available + esClient = newEsClient + // Get all the API Keys, for that specific client, on the reconciled cluster. + getCrossClusterAPIKeys, err := esClient.GetCrossClusterAPIKeys(ctx, "eck-*") + if err != nil { + return reconcile.Result{}, err + } + activeAPIKeys = getCrossClusterAPIKeys + } + + // apiKeyReconciledRemoteClusters is used to track all the client clusters for which API keys have already been reconciled. + // This is used to garbage collect API keys for clusters which have been deleted and are not in expectedRemoteClusters. + apiKeyReconciledRemoteClusters := sets.New[types.NamespacedName]() + + // Main loop to: + // 1. Create or update expected remote CA. + // 2. Create or update API keys and keystores. + for remoteEsKey, remoteClusters := range expectedRemoteClusters { + // Get the remote/client Elasticsearch cluster associated with this local/reconciled cluster. remoteEs := &esv1.Elasticsearch{} if err := r.Client.Get(ctx, remoteEsKey, remoteEs); err != nil { if errors.IsNotFound(err) { - // Remote cluster does not exist, skip it + // Remote cluster does not exist, invalidate API keys for that client cluster. + apiKeyReconciledRemoteClusters.Insert(remoteEsKey) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, nil, esClient)) continue } return reconcile.Result{}, err } + log := log.WithValues( + "local_namespace", localEs.Namespace, + "local_name", localEs.Name, + "remote_namespace", remoteEs.Namespace, + "remote_name", remoteEs.Name, + ) accessAllowed, err := isRemoteClusterAssociationAllowed(ctx, r.accessReviewer, localEs, remoteEs, r.recorder) if err != nil { return reconcile.Result{}, err } // if the remote CA exists but isn't allowed anymore, it will be deleted next if !accessAllowed { + // Remove from the expected remote cluster to clean up local keystore. + delete(expectedRemoteClusters, remoteEsKey) + // Invalidate API keys for that client cluster. + apiKeyReconciledRemoteClusters.Insert(remoteEsKey) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, nil, esClient)) continue } - delete(remoteClustersInvolved, remoteEsKey) + delete(associatedRemoteCAs, remoteEsKey) results.WithResults(createOrUpdateCertificateAuthorities(ctx, r, localEs, remoteEs)) if results.HasError() { return results.Aggregate() } + + // RCS2, first check that both the reconciled and the client clusters are compatible. + clientClusterSupportClusterAPIKeys, err := remoteEs.SupportRemoteClusterAPIKeys() + if err != nil { + results.WithError(err) + continue + } + + if !clientClusterSupportClusterAPIKeys.IsSet() { + log.Info("Client cluster version is not available in status yet, skipping API keys reconciliation") + continue + } + + if !localClusterSupportClusterAPIKeys.IsSet() { + log.Info("Cluster version is not available in status yet, skipping API keys reconciliation") + continue + } + + if clientClusterSupportClusterAPIKeys.IsFalse() && localClusterSupportClusterAPIKeys.IsTrue() { + err := fmt.Errorf("client cluster %s/%s is running version %s which does not support remote cluster keys", remoteEs.Namespace, remoteEs.Name, remoteEs.Spec.Version) + log.Error(err, "cannot configure remote cluster settings") + continue + } + // Reconcile the API Keys. + apiKeyReconciledRemoteClusters.Insert(remoteEsKey) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, remoteClusters, esClient)) + } + + if localClusterSupportClusterAPIKeys.IsTrue() { + // ************************************************************** + // Delete orphaned API keys from clusters which have been deleted + // ************************************************************** + for _, activeAPIKey := range activeAPIKeys.APIKeys { + clientCluster, err := activeAPIKey.GetElasticsearchName() + if err != nil { + results.WithError(err) + continue + } + if _, exists := apiKeyReconciledRemoteClusters[clientCluster]; exists { + // API keys for that client cluster have already been reconciled, skip. + continue + } + // This API key in the local cluster state belongs to an unknown cluster which is not expected and has not been reconciled. + log.Info(fmt.Sprintf("Invalidating API key %s which belongs to unknown cluster %s", activeAPIKey.Name, clientCluster)) + results.WithError(esClient.InvalidateCrossClusterAPIKey(ctx, activeAPIKey.Name)) + } + + // ********************************************* + // Delete unexpected keys in the local keystore. + // ********************************************* + expectedAliases := expectedAliases(localEs, expectedRemoteClusters) + apiKeyStore, err := LoadAPIKeyStore(ctx, r.Client, localEs) + if err != nil { + return results.WithError(err).Aggregate() + } + for alias := range apiKeyStore.aliases { + if expectedAliases.Has(alias) { + // Expected alias + continue + } + // Unexpected + log.Info(fmt.Sprintf("Removing unexpected remote API key %s", alias)) + apiKeyStore.Delete(alias) + } + results.WithError(apiKeyStore.Save(ctx, r.Client, localEs)) } // Delete existing but not expected remote CA - for toDelete := range remoteClustersInvolved { + for toDelete := range associatedRemoteCAs { log.V(1).Info("Deleting remote CA", "local_namespace", localEs.Namespace, "local_name", localEs.Name, @@ -199,20 +320,41 @@ func doReconcile( return results.WithResult(association.RequeueRbacCheck(r.accessReviewer)).Aggregate() } +func expectedAliases( + localCluster *esv1.Elasticsearch, + expectedRemoteCluster map[types.NamespacedName][]esv1.RemoteCluster, +) sets.Set[string] { + aliases := sets.New[string]() + for _, remoteCluster := range localCluster.Spec.RemoteClusters { + clientClusterNamespacedName := remoteCluster.ElasticsearchRef.WithDefaultNamespace(localCluster.Namespace).NamespacedName() + if _, ok := expectedRemoteCluster[clientClusterNamespacedName]; !ok { + // Not expected, might have been filtered by RBAC rules + continue + } + if remoteCluster.APIKey == nil { + // Not using remote cluster server. + continue + } + aliases.Insert(remoteCluster.Name) + } + return aliases +} + func caCertMissingError(cluster types.NamespacedName) string { return fmt.Sprintf("Cannot find CA certificate cluster %s/%s", cluster.Namespace, cluster.Name) } -// getExpectedRemoteClusters returns all the remote cluster keys for which a remote ca should created -// The CA certificates must be copied from the remote cluster to the local one and vice versa +// getExpectedRemoteClusters returns all the remote cluster keys for which a remote ca and an API Key should be created. +// The CA certificates must be copied from the remote cluster to the local one and vice versa. +// The API Key is created in the remote cluster and injected in the keystore of the local cluster. func getExpectedRemoteClusters( ctx context.Context, c k8s.Client, associatedEs *esv1.Elasticsearch, -) (map[types.NamespacedName]struct{}, error) { - span, _ := apm.StartSpan(ctx, "get_expected_remote_ca", tracing.SpanTypeApp) +) (map[types.NamespacedName][]esv1.RemoteCluster, error) { + span, _ := apm.StartSpan(ctx, "get_expected_remote_clusters", tracing.SpanTypeApp) defer span.End() - expectedRemoteClusters := make(map[types.NamespacedName]struct{}) + expectedRemoteClusters := make(map[types.NamespacedName][]esv1.RemoteCluster) // Add remote clusters declared in the Spec for _, remoteCluster := range associatedEs.Spec.RemoteClusters { @@ -220,7 +362,7 @@ func getExpectedRemoteClusters( continue } esRef := remoteCluster.ElasticsearchRef.WithDefaultNamespace(associatedEs.Namespace) - expectedRemoteClusters[esRef.NamespacedName()] = struct{}{} + expectedRemoteClusters[esRef.NamespacedName()] = nil } var list esv1.ElasticsearchList @@ -238,7 +380,8 @@ func getExpectedRemoteClusters( esRef := remoteCluster.ElasticsearchRef.WithDefaultNamespace(es.Namespace) if esRef.Namespace == associatedEs.Namespace && esRef.Name == associatedEs.Name { - expectedRemoteClusters[k8s.ExtractNamespacedName(&es)] = struct{}{} + clientClusterName := k8s.ExtractNamespacedName(&es) + expectedRemoteClusters[clientClusterName] = append(expectedRemoteClusters[clientClusterName], remoteCluster) } } } @@ -246,13 +389,13 @@ func getExpectedRemoteClusters( return expectedRemoteClusters, nil } -// remoteClustersInvolvedWith returns for a given Elasticsearch cluster all the Elasticsearch keys for which +// getAssociatedRemoteCAs returns for a given Elasticsearch cluster all the Elasticsearch keys for which // the remote certificate authorities have been copied, i.e. all the other Elasticsearch clusters for which this cluster // has been involved in a remote cluster association. // In order to get all of them we: // 1. List all the remote CA copied locally. // 2. List all the other Elasticsearch clusters for which the CA of the given cluster has been copied. -func remoteClustersInvolvedWith( +func getAssociatedRemoteCAs( ctx context.Context, c k8s.Client, es types.NamespacedName, diff --git a/pkg/controller/remotecluster/controller_test.go b/pkg/controller/remotecluster/controller_test.go new file mode 100644 index 0000000000..b453de4041 --- /dev/null +++ b/pkg/controller/remotecluster/controller_test.go @@ -0,0 +1,970 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "reflect" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/license" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/watches" + esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/net" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac" +) + +func TestRemoteCluster_Reconcile(t *testing.T) { + type fields struct { + clusters []client.Object + accessReviewer rbac.AccessReviewer + licenseChecker license.Checker + } + type args struct { + request reconcile.Request + } + type wantEsAPICalls struct { + getCrossClusterAPIKeys []string + invalidateCrossClusterAPIKey []string + crossClusterAPIKeyCreateRequests []esclient.CrossClusterAPIKeyCreateRequest + updateCrossClusterAPIKey map[string]esclient.CrossClusterAPIKeyUpdateRequest + } + tests := []struct { + name string + fields fields + args args + + expectedCASecrets []*corev1.Secret + unexpectedSecrets []types.NamespacedName + want reconcile.Result + wantErr bool + + // API keys fake client and expected resources. + fakeESClient *fakeESClient + wantEsAPICalls wantEsAPICalls + expectedKeystoreSecrets []*corev1.Secret + }{ + { + name: "Simple remote cluster ns1/es1 -> ns2/es2, ns1", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "[No API Keys] Bi-directional remote cluster ns1/es1 <-> ns2/es2", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").withRemoteCluster("ns1", "es1").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + // The test below simulates the following situation: + // * ns1/es1 is the cluster reconciled. + // * ns1/es2 is the client cluster. + name: "With API Keys, simple topology", + fields: fields{ + clusters: slices.Concat( + // Clusters + newClusterBuilder("ns1", "es1", "8.15.0").build(), + newClusterBuilder("ns1", "es2", "8.15.0"). + // es2 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + []client.Object{ + // Certificates + fakePublicCa("ns1", "es1"), + fakePublicCa("ns1", "es2"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + // reconciled cluster is es1 + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns1", "es2"), + remoteCa("ns1", "es2", "ns1", "es1"), + }, + wantEsAPICalls: wantEsAPICalls{ + getCrossClusterAPIKeys: []string{"eck-*"}, + crossClusterAPIKeyCreateRequests: []esclient.CrossClusterAPIKeyCreateRequest{ + { + Name: "eck-ns1-es2-generated-ns1-es1-0-with-api-key", + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es2", + "elasticsearch.k8s.elastic.co/namespace": "ns1", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + }, + }, + expectedKeystoreSecrets: []*corev1.Secret{ + { + // Keystore for es2 must be updated. + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"eck-ns1-es2-generated-ns1-es1-0-with-api-key-1"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns1", + Name: "es2-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-eck-ns1-es2-generated-ns1-es1-0-with-api-key-1"), + }, + }, + }, + }, + { + // The test below simulates the following situation: + // * ns1/es1 is the cluster reconciled. + // * There are 3 remote cluster accessing ns1/es1: + // * ns2/es2: new cluster, API key must be created + // * ns3/es3: new cluster, API key must be created + // * ns4/es4: existing remote cluster: one key must be updated, the other one must be deleted. + // * ns/es5: this cluster no long exists, key must be deleted. + name: "With API Keys, complex topology", + fields: fields{ + clusters: slices.Concat( + // Clusters + newClusterBuilder("ns1", "es1", "8.15.0"). + // es1 -> es2 + withAPIKey("ns2", "es2", &esv1.RemoteClusterAPIKey{}). + // es1 -> es3 + withAPIKey("ns3", "es3", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns2", "es2", "8.15.0"). + // es2 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns3", "es3", "8.15.0"). + // es3 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns4", "es4", "8.15.0"). + // es4-> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + []client.Object{ + // Certificates + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + fakePublicCa("ns3", "es3"), + fakePublicCa("ns4", "es4"), + // Assume that ns2/es2 has already a key in its keystore, the goal is to test that the existing keys are not altered. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns2", + Name: "es2-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + // Keystore for ns4/es4, key already exists, this Secret is not expected to be modified. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"apikey-from-es4-to-es1"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es4", + }, + Namespace: "ns4", + Name: "es4-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + // Assume that ns1/es1 keystore already exists, unexpected keys should be removed. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns2-es2-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-ns4-es4-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es4"},"api-key-to-non-existent-alias":{"namespace":"nsx","name":"esx","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es1", + }, + Namespace: "ns1", + Name: "es1-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns2-es2-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-ns4-es4-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + // The key below should be removed + "cluster.remote.api-key-to-non-existent-alias.credentials": []byte("encoded-key-for-api-for-non-existent-cluster"), + }, + }, + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + // reconciled cluster is es1 + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + fakeESClient: &fakeESClient{ + existingCrossClusterAPIKeys: esclient.CrossClusterAPIKeyList{ + // API key for es4 already exists but with a wrong hash we should expect an update. + APIKeys: []esclient.CrossClusterAPIKey{ + { + ID: "apikey-from-es4-to-es1", + Name: "eck-ns4-es4-generated-ns1-es1-0-with-api-key", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + // The key below belongs to a cluster which no longer exists, it must be invalidated. + { + ID: "apikey-from-es5-to-es1", + Name: "eck-ns5-es5-generated-ns1-es1-0-with-api-key", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es5", + "elasticsearch.k8s.elastic.co/namespace": "ns5", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + // The key below belongs to the existing cluster es4 which is no longer referencing es1 using that alias, it must be invalidated. + { + ID: "apikey-from-es4-to-es1-old-alias", + Name: "eck-ns4-es4-to-ns1-es1-0-old-alias", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + }, + }, + wantEsAPICalls: wantEsAPICalls{ + getCrossClusterAPIKeys: []string{"eck-*"}, + invalidateCrossClusterAPIKey: []string{"eck-ns4-es4-to-ns1-es1-0-old-alias", "eck-ns5-es5-generated-ns1-es1-0-with-api-key"}, + updateCrossClusterAPIKey: map[string]esclient.CrossClusterAPIKeyUpdateRequest{ + "apikey-from-es4-to-es1": { + RemoteClusterAPIKey: esv1.RemoteClusterAPIKey{}, + Metadata: map[string]any{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + // We expect 2 keys to be created for ns1/es1 + crossClusterAPIKeyCreateRequests: []esclient.CrossClusterAPIKeyCreateRequest{ + { + Name: "eck-ns2-es2-generated-ns1-es1-0-with-api-key", + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es2", + "elasticsearch.k8s.elastic.co/namespace": "ns2", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + { + Name: "eck-ns3-es3-generated-ns1-es1-0-with-api-key", + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es3", + "elasticsearch.k8s.elastic.co/namespace": "ns3", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + }, + }, + expectedKeystoreSecrets: []*corev1.Secret{ + // Unexpected keys in es1 keystore must be removed. + { + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns2-es2-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es1", + }, + Namespace: "ns1", + Name: "es1-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns2-es2-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + { + // Keystore for es2 must be updated. + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"},"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"eck-ns2-es2-generated-ns1-es1-0-with-api-key-1"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns2", + Name: "es2-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-eck-ns2-es2-generated-ns1-es1-0-with-api-key-1"), + }, + }, + { + // Keystore for es3 must be created. + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"eck-ns3-es3-generated-ns1-es1-0-with-api-key-2"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es3", + }, + Namespace: "ns3", + Name: "es3-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-eck-ns3-es3-generated-ns1-es1-0-with-api-key-2"), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"apikey-from-es4-to-es1"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es4", + }, + Namespace: "ns4", + Name: "es4-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + }, + }, + { + // Same test as above but the associations are no longer permitted. + // * ns1/es1 is the cluster reconciled. + // * There are 3 remote cluster accessing ns1/es1: + // * ns2/es2: new cluster, API key must not be created + // * ns3/es3: new cluster, API key must not be created + // * ns4/es4: existing remote cluster: 2 keys must be deleted. + // * ns/es5: this cluster no long exists, key must be deleted. + name: "With API Keys: associations are no longer permitted", + fields: fields{ + clusters: slices.Concat( + // Clusters + newClusterBuilder("ns1", "es1", "8.15.0"). + // es1 -> es2 + withAPIKey("ns2", "es2", &esv1.RemoteClusterAPIKey{}). + // es1 -> es3 + withAPIKey("ns3", "es3", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns2", "es2", "8.15.0"). + // es2 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns3", "es3", "8.15.0"). + // es3 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns4", "es4", "8.15.0"). + // es4-> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + []client.Object{ + // Certificates + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + fakePublicCa("ns3", "es3"), + fakePublicCa("ns4", "es4"), + // Assume that ns2/es2 has already a key in its keystore, the goal is to test that the existing keys are not altered. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns2", + Name: "es2-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + // Keystore for ns4/es4, key already exists, this Secret must be deleted since association is no longer permitted. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"apikey-from-es4-to-es1"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es4", + }, + Namespace: "ns4", + Name: "es4-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + // Assume that ns1/es1 keystore already exists, unexpected keys should be removed. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns2-es2-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-ns4-es4-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es4"},"api-key-to-non-existent-alias":{"namespace":"nsx","name":"esx","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es1", + }, + Namespace: "ns1", + Name: "es1-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns2-es2-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-ns4-es4-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + // The key below should be removed + "cluster.remote.api-key-to-non-existent-alias.credentials": []byte("encoded-key-for-api-for-non-existent-cluster"), + }, + }, + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: false}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + // reconciled cluster is es1 + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + fakeESClient: &fakeESClient{ + existingCrossClusterAPIKeys: esclient.CrossClusterAPIKeyList{ + // API key for es4 already exists but with a wrong hash we should expect an update. + APIKeys: []esclient.CrossClusterAPIKey{ + { + ID: "apikey-from-es4-to-es1", + Name: "eck-ns4-es4-generated-ns1-es1-0-with-api-key", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + // The key below belongs to a cluster which no longer exists, it must be invalidated. + { + ID: "apikey-from-es5-to-es1", + Name: "eck-ns5-es5-generated-ns1-es1-0-with-api-key", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es5", + "elasticsearch.k8s.elastic.co/namespace": "ns5", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + // The key below belongs to the existing cluster es4 which is no longer referencing es1 using that alias, it must be invalidated. + { + ID: "apikey-from-es4-to-es1-old-alias", + Name: "eck-ns4-es4-to-ns1-es1-0-old-alias", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + }, + }, + wantEsAPICalls: wantEsAPICalls{ + getCrossClusterAPIKeys: []string{"eck-*"}, + invalidateCrossClusterAPIKey: []string{"eck-ns4-es4-generated-ns1-es1-0-with-api-key", "eck-ns4-es4-to-ns1-es1-0-old-alias", "eck-ns5-es5-generated-ns1-es1-0-with-api-key"}, + // No update allowed. + updateCrossClusterAPIKey: nil, + // No creation allowed. + crossClusterAPIKeyCreateRequests: nil, + }, + unexpectedSecrets: []types.NamespacedName{ + {Namespace: "ns1", Name: "es1-es-remote-api-keys"}, + {Namespace: "ns3", Name: "es3-es-remote-api-keys"}, + {Namespace: "ns4", Name: "es4-es-remote-api-keys"}, + }, + expectedKeystoreSecrets: []*corev1.Secret{ + { + // Keystore for es2 must not be updated with es1 key. + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns2", + Name: "es2-es-remote-api-keys", + }, + Data: map[string][]byte{ + // This entry is going to be removed once ns2/es2 is reconciled. + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + }, + }, + { + name: "Deleted remote cluster", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + unexpectedSecrets: []types.NamespacedName{ + { + Namespace: "ns1", + Name: remoteCASecretName("es1", types.NamespacedName{ + Namespace: "ns2", + Name: "es2", + }), + }, + { + Namespace: "ns2", + Name: remoteCASecretName("es2", types.NamespacedName{ + Namespace: "ns1", + Name: "es1", + }), + }, + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "CA content has been updated, remote ca must be reconciled", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + withDataCert(remoteCa("ns1", "es1", "ns2", "es2"), []byte("foo")), + withDataCert(remoteCa("ns2", "es2", "ns1", "es1"), []byte("bar")), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + // ns1/es1 has been deleted - all related secrets in other namespaces must be deleted + name: "Deleted cluster", + fields: fields{ + clusters: slices.Concat( + // ns2/es2 + newClusterBuilder("ns2", "es2", "7.0.0").withRemoteCluster("ns1", "es1").build(), + // ns3/es3 + newClusterBuilder("ns3", "es3", "7.0.0").withRemoteCluster("ns1", "es1").build(), + []client.Object{ + fakePublicCa("ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + fakePublicCa("ns3", "es3"), + remoteCa("ns3", "es3", "ns1", "es1"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + unexpectedSecrets: []types.NamespacedName{ + { + Namespace: "ns3", + Name: remoteCASecretName("es3", types.NamespacedName{ + Namespace: "ns1", + Name: "es1", + }), + }, + { + Namespace: "ns2", + Name: remoteCASecretName("es2", types.NamespacedName{ + Namespace: "ns1", + Name: "es1", + }), + }, + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "No enterprise license, remote ca are not created", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: false}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + unexpectedSecrets: []types.NamespacedName{ + { + Namespace: "ns1", + Name: remoteCASecretName("es1", types.NamespacedName{ + Namespace: "ns2", + Name: "es2", + }), + }, + { + Namespace: "ns2", + Name: remoteCASecretName("es2", types.NamespacedName{ + Namespace: "ns1", + Name: "es1", + }), + }, + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "No enterprise license, existing remote ca are left untouched", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: false}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "Association is not allowed, existing remote ca are removed", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: false}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + unexpectedSecrets: []types.NamespacedName{ + { + Namespace: "ns1", + Name: remoteCASecretName("es1", types.NamespacedName{Namespace: "ns2", Name: "es2"}), + }, + { + Namespace: "ns2", + Name: remoteCASecretName("es2", types.NamespacedName{Namespace: "ns1", Name: "es1"}), + }, + }, + want: reconcile.Result{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := watches.NewDynamicWatches() + fakeESClient := &fakeESClient{} + if tt.fakeESClient != nil { + fakeESClient = tt.fakeESClient + } + k8sClient := k8s.NewFakeClient(tt.fields.clusters...) + r := &ReconcileRemoteClusters{ + Client: k8sClient, + accessReviewer: tt.fields.accessReviewer, + watches: w, + licenseChecker: tt.fields.licenseChecker, + recorder: record.NewFakeRecorder(10), + esClientProvider: func(_ context.Context, _ k8s.Client, _ net.Dialer, _ esv1.Elasticsearch) (esclient.Client, error) { + return fakeESClient, nil + }, + } + fakeCtx := context.Background() + got, err := r.Reconcile(fakeCtx, tt.args.request) + if (err != nil) != tt.wantErr { + t.Errorf("ReconcileRemoteCa.Reconcile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ReconcileRemoteCa.Reconcile() = %v, want %v", got, tt.want) + } + // Check that expected secrets are here + for _, expectedSecret := range tt.expectedCASecrets { + var actualSecret corev1.Secret + assert.NoError(t, r.Client.Get(context.Background(), types.NamespacedName{Namespace: expectedSecret.Namespace, Name: expectedSecret.Name}, &actualSecret)) + // Compare content + actualCa, ok := actualSecret.Data[certificates.CAFileName] + assert.True(t, ok) + assert.Equal(t, expectedSecret.Data[certificates.CAFileName], actualCa) + // Compare labels + assert.NotNil(t, actualSecret.Labels) + assert.Equal(t, expectedSecret.Labels, actualSecret.Labels) + } + // Check that unexpected secrets does not exist + for _, unexpectedSecret := range tt.unexpectedSecrets { + var actualSecret corev1.Secret + err := r.Client.Get(context.Background(), types.NamespacedName{Namespace: unexpectedSecret.Namespace, Name: unexpectedSecret.Name}, &actualSecret) + assert.True(t, apierrors.IsNotFound(err), "unexpected Secret %s/%s", unexpectedSecret.Namespace, unexpectedSecret.Name) + } + // Fake ES client assertions + assert.ElementsMatch(t, tt.wantEsAPICalls.getCrossClusterAPIKeys, fakeESClient.getCrossClusterAPIKeys, "unexpected calls to GetCrossClusterAPIKeys") + assert.ElementsMatch(t, tt.wantEsAPICalls.invalidateCrossClusterAPIKey, fakeESClient.invalidateCrossClusterAPIKey, "unexpected calls to InvalidateCrossClusterAPIKey") + assert.ElementsMatch( + t, + tt.wantEsAPICalls.crossClusterAPIKeyCreateRequests, + fakeESClient.crossClusterAPIKeyCreateRequests, + "unexpected calls to CreateCrossClusterAPIKey\n%s\n", cmp.Diff(tt.wantEsAPICalls.crossClusterAPIKeyCreateRequests, fakeESClient.crossClusterAPIKeyCreateRequests), + ) + assert.Equal( + t, + tt.wantEsAPICalls.updateCrossClusterAPIKey, + fakeESClient.updateCrossClusterAPIKey, + "unexpected calls to UpdateCrossClusterAPIKey\n%s\n", cmp.Diff(tt.wantEsAPICalls.updateCrossClusterAPIKey, fakeESClient.updateCrossClusterAPIKey), + ) + + // Keystore assertions + for _, expectedSecret := range tt.expectedKeystoreSecrets { + actualSecret := &corev1.Secret{} + if err := k8sClient.Get(fakeCtx, types.NamespacedName{ + Namespace: expectedSecret.Namespace, + Name: expectedSecret.Name, + }, actualSecret); err != nil { + t.Fatalf("error while retrieving keystore %s/%s: %v", expectedSecret.Namespace, expectedSecret.Name, err) + } + if diff := cmp.Diff(expectedSecret.Labels, actualSecret.Labels); len(diff) > 0 { + t.Errorf("unexpected labels on Secret %s/%s:\n%s\n", expectedSecret.Namespace, expectedSecret.Name, diff) + } + if diff := cmp.Diff(expectedSecret.Annotations, actualSecret.Annotations); len(diff) > 0 { + t.Errorf("unexpected annotations on Secret %s/%s:\n%s\n", expectedSecret.Namespace, expectedSecret.Name, diff) + } + if diff := cmp.Diff(expectedSecret.Data, actualSecret.Data); len(diff) > 0 { + t.Errorf("unexpected data in Secret %s/%s:\n%s\n", expectedSecret.Namespace, expectedSecret.Name, diff) + } + } + }) + } +} diff --git a/pkg/controller/remotecluster/fixtures.go b/pkg/controller/remotecluster/fixtures.go new file mode 100644 index 0000000000..37214dbb05 --- /dev/null +++ b/pkg/controller/remotecluster/fixtures.go @@ -0,0 +1,198 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/certificates/transport" + esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" +) + +// -- Fake ES client + +type fakeESClient struct { + esclient.Client + + // fake responses + getCrossClusterAPIKeys []string + + // recorded requests + invalidateCrossClusterAPIKey []string + crossClusterAPIKeyCreateRequests []esclient.CrossClusterAPIKeyCreateRequest + existingCrossClusterAPIKeys esclient.CrossClusterAPIKeyList + updateCrossClusterAPIKey map[string]esclient.CrossClusterAPIKeyUpdateRequest +} + +func (f *fakeESClient) CreateCrossClusterAPIKey(ctx context.Context, crossClusterAPIKeyCreateRequest esclient.CrossClusterAPIKeyCreateRequest) (esclient.CrossClusterAPIKeyCreateResponse, error) { + f.crossClusterAPIKeyCreateRequests = append(f.crossClusterAPIKeyCreateRequests, crossClusterAPIKeyCreateRequest) + return esclient.CrossClusterAPIKeyCreateResponse{ + ID: fmt.Sprintf("%s-%d", crossClusterAPIKeyCreateRequest.Name, len(f.crossClusterAPIKeyCreateRequests)), + Name: crossClusterAPIKeyCreateRequest.Name, + APIKey: fmt.Sprintf("api-key-for-%s-%d", crossClusterAPIKeyCreateRequest.Name, len(f.crossClusterAPIKeyCreateRequests)), + Encoded: fmt.Sprintf("encoded-key-for-%s-%d", crossClusterAPIKeyCreateRequest.Name, len(f.crossClusterAPIKeyCreateRequests)), + }, nil +} + +func (f *fakeESClient) GetCrossClusterAPIKeys(_ context.Context, name string) (esclient.CrossClusterAPIKeyList, error) { + f.getCrossClusterAPIKeys = append(f.getCrossClusterAPIKeys, name) + return f.existingCrossClusterAPIKeys, nil +} + +func (f *fakeESClient) InvalidateCrossClusterAPIKey(_ context.Context, name string) error { + f.invalidateCrossClusterAPIKey = append(f.invalidateCrossClusterAPIKey, name) + return nil +} + +func (f *fakeESClient) UpdateCrossClusterAPIKey(_ context.Context, name string, updateRequest esclient.CrossClusterAPIKeyUpdateRequest) (esclient.CrossClusterAPIKeyUpdateResponse, error) { + if f.updateCrossClusterAPIKey == nil { + f.updateCrossClusterAPIKey = make(map[string]esclient.CrossClusterAPIKeyUpdateRequest) + } + f.updateCrossClusterAPIKey[name] = updateRequest + return esclient.CrossClusterAPIKeyUpdateResponse{}, nil +} + +// -- Fake cluster builder + +type clusterBuilder struct { + name, namespace, version string + remoteClusters []esv1.RemoteCluster +} + +func newClusterBuilder(namespace, name, version string) *clusterBuilder { + return &clusterBuilder{ + name: name, + namespace: namespace, + version: version, + } +} + +func (cb *clusterBuilder) withRemoteCluster(namespace, name string) *clusterBuilder { + cb.remoteClusters = append(cb.remoteClusters, + esv1.RemoteCluster{ + Name: fmt.Sprintf("generated-%s-%s-%d", namespace, name, len(cb.remoteClusters)), + ElasticsearchRef: commonv1.LocalObjectSelector{ + Name: name, + Namespace: namespace, + }, + }) + return cb +} + +func (cb *clusterBuilder) withAPIKey(namespace, name string, apiKey *esv1.RemoteClusterAPIKey) *clusterBuilder { + cb.remoteClusters = append(cb.remoteClusters, + esv1.RemoteCluster{ + Name: fmt.Sprintf("generated-%s-%s-%d-with-api-key", namespace, name, len(cb.remoteClusters)), + ElasticsearchRef: commonv1.LocalObjectSelector{ + Name: name, + Namespace: namespace, + }, + APIKey: apiKey, + }) + return cb +} + +func (cb *clusterBuilder) build() []client.Object { + remoteClusters := make([]esv1.RemoteCluster, len(cb.remoteClusters)) + for i, remoteCluster := range cb.remoteClusters { + remoteCluster := remoteCluster.DeepCopy() + remoteClusters[i] = *remoteCluster + } + return []client.Object{ + &esv1.Elasticsearch{ + ObjectMeta: v1.ObjectMeta{ + Namespace: cb.namespace, + Name: cb.name, + }, + Spec: esv1.ElasticsearchSpec{ + Version: cb.version, + RemoteClusters: remoteClusters, + }, + Status: esv1.ElasticsearchStatus{ + AvailableNodes: 3, + Version: cb.version, + }, + }, + &corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Namespace: cb.namespace, + Name: fmt.Sprintf("es-%s-%s-1", cb.namespace, cb.name), + Labels: map[string]string{ + label.ClusterNameLabelName: cb.name, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + } +} + +type fakeAccessReviewer struct { + allowed bool + err error +} + +func (f *fakeAccessReviewer) AccessAllowed(_ context.Context, _ string, _ string, _ runtime.Object) (bool, error) { + return f.allowed, f.err +} + +func fakePublicCa(namespace, name string) *corev1.Secret { + namespacedName := types.NamespacedName{ + Name: name, + Namespace: namespace, + } + transportPublicCertKey := transport.PublicCertsSecretRef(namespacedName) + return &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Namespace: transportPublicCertKey.Namespace, + Name: transportPublicCertKey.Name, + }, + Data: map[string][]byte{ + certificates.CAFileName: []byte(namespacedName.String()), + }, + } +} + +// remoteCa builds an expected remote Ca +func remoteCa(localNamespace, localName, remoteNamespace, remoteName string) *corev1.Secret { + remoteNamespacedName := types.NamespacedName{ + Name: remoteName, + Namespace: remoteNamespace, + } + return &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Namespace: localNamespace, + Name: remoteCASecretName(localName, remoteNamespacedName), + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-ca", + "elasticsearch.k8s.elastic.co/cluster-name": localName, + "elasticsearch.k8s.elastic.co/remote-cluster-name": remoteName, + "elasticsearch.k8s.elastic.co/remote-cluster-namespace": remoteNamespace, + }, + }, + Data: map[string][]byte{ + certificates.CAFileName: []byte(remoteNamespacedName.String()), + }, + } +} + +func withDataCert(caSecret *corev1.Secret, newCa []byte) *corev1.Secret { + caSecret.Data[certificates.CAFileName] = newCa + return caSecret +} diff --git a/pkg/controller/remotecluster/keystore.go b/pkg/controller/remotecluster/keystore.go new file mode 100644 index 0000000000..7529bdc361 --- /dev/null +++ b/pkg/controller/remotecluster/keystore.go @@ -0,0 +1,219 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + + "k8s.io/apimachinery/pkg/util/sets" + + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/labels" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" +) + +const ( + aliasesAnnotationName = "elasticsearch.k8s.elastic.co/remote-cluster-api-keys" + + remoteClusterAPIKeysType = "remote-cluster-api-keys" +) + +var ( + credentialsSecretSettingsRegEx = regexp.MustCompile(`^cluster\.remote\.([\w-]+)\.credentials$`) +) + +type APIKeyStore struct { + // aliases maps cluster aliased with the expected key ID + aliases map[string]AliasValue + // keys maps the ID of an API Key (not its name), to the encoded cross-cluster API key. + keys map[string]string +} + +type AliasValue struct { + // Namespace of the remote cluster. + Namespace string `json:"namespace"` + // Name of the remote cluster. + Name string `json:"name"` + // ID is the key ID. + ID string `json:"id"` +} + +func (aks *APIKeyStore) KeyIDFor(alias string) string { + if aks == nil { + return "" + } + return aks.aliases[alias].ID +} + +func LoadAPIKeyStore(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearch) (*APIKeyStore, error) { + secretName := types.NamespacedName{ + Name: esv1.RemoteAPIKeysSecretName(owner.Name), + Namespace: owner.Namespace, + } + // Attempt to read the Secret + keyStoreSecret := &corev1.Secret{} + if err := c.Get(ctx, secretName, keyStoreSecret); err != nil { + if errors.IsNotFound(err) { + ulog.FromContext(ctx).V(1).Info("No APIKeyStore Secret found", + "namespace", owner.Namespace, + "es_name", owner.Name, + ) + // Return an empty store + return &APIKeyStore{}, nil + } + } + + // Read the key aliased + aliases := make(map[string]AliasValue) + if aliasesAnnotation, ok := keyStoreSecret.Annotations[aliasesAnnotationName]; ok { + if err := json.Unmarshal([]byte(aliasesAnnotation), &aliases); err != nil { + return nil, err + } + } + + // Read the current encoded cross-cluster API keys. + keys := make(map[string]string) + for settingName, encodedAPIKey := range keyStoreSecret.Data { + strings := credentialsSecretSettingsRegEx.FindStringSubmatch(settingName) + if len(strings) != 2 { + ulog.FromContext(ctx).V(1).Info( + fmt.Sprintf("Unknown remote cluster credential setting: %s", settingName), + "namespace", owner.Namespace, + "es_name", owner.Name, + ) + continue + } + keys[strings[1]] = string(encodedAPIKey) + } + return &APIKeyStore{ + aliases: aliases, + keys: keys, + }, nil +} + +func (aks *APIKeyStore) Update(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue string) *APIKeyStore { + if aks.aliases == nil { + aks.aliases = make(map[string]AliasValue) + } + aks.aliases[alias] = AliasValue{ + Namespace: remoteClusterNamespace, + Name: remoteClusterName, + ID: keyID, + } + if aks.keys == nil { + aks.keys = make(map[string]string) + } + aks.keys[alias] = encodedKeyValue + return aks +} + +func (aks *APIKeyStore) Aliases() []string { + if aks == nil { + return nil + } + aliases := make([]string, len(aks.aliases)) + i := 0 + for alias := range aks.aliases { + aliases[i] = alias + i++ + } + return aliases +} + +func (aks *APIKeyStore) Delete(alias string) *APIKeyStore { + delete(aks.aliases, alias) + delete(aks.keys, alias) + return aks +} + +const ( + credentialsKeyFormat = "cluster.remote.%s.credentials" +) + +func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearch) error { + secretName := types.NamespacedName{ + Name: esv1.RemoteAPIKeysSecretName(owner.Name), + Namespace: owner.Namespace, + } + if aks.IsEmpty() { + // Check if the Secret does exist. + currentSecret := corev1.Secret{} + if err := c.Get(ctx, secretName, ¤tSecret); err != nil { + if errors.IsNotFound(err) { + // Secret does not exist. + return nil + } + return err + } + // Delete the Secret we just detected above. + if err := c.Delete(ctx, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName.Name, Namespace: secretName.Namespace}}, + &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: ¤tSecret.UID}}, + ); err != nil { + return err + } + return nil + } + + aliases, err := json.Marshal(aks.aliases) + if err != nil { + return err + } + data := make(map[string][]byte, len(aks.keys)) + for k, v := range aks.keys { + data[fmt.Sprintf(credentialsKeyFormat, k)] = []byte(v) + } + expectedLabels := labels.AddCredentialsLabel(label.NewLabels(k8s.ExtractNamespacedName(owner))) + expectedLabels[commonv1.TypeLabelName] = remoteClusterAPIKeysType + expected := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName.Name, + Namespace: secretName.Namespace, + Annotations: map[string]string{ + aliasesAnnotationName: string(aliases), + }, + Labels: expectedLabels, + }, + Data: data, + } + if _, err := reconciler.ReconcileSecret(ctx, c, expected, owner); err != nil { + return err + } + return nil +} + +func (aks *APIKeyStore) IsEmpty() bool { + if aks == nil { + return true + } + return len(aks.aliases) == 0 +} + +func (aks *APIKeyStore) ForCluster(namespace string, name string) sets.Set[string] { + aliases := sets.New[string]() + if aks == nil { + return aliases + } + for alias, c := range aks.aliases { + if c.Name == name && c.Namespace == namespace { + aliases.Insert(alias) + } + } + return aliases +} diff --git a/pkg/controller/remotecluster/keystore_test.go b/pkg/controller/remotecluster/keystore_test.go new file mode 100644 index 0000000000..90ca9cab41 --- /dev/null +++ b/pkg/controller/remotecluster/keystore_test.go @@ -0,0 +1,282 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" +) + +const ( + testNamespace = "ns1" +) + +var ( + es = &esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myes", + Namespace: testNamespace, + UID: uuid.NewUUID(), + Generation: 42, + }, + Spec: esv1.ElasticsearchSpec{}, + } +) + +func TestLoadAPIKeyStore(t *testing.T) { + type args struct { + c k8s.Client + owner *esv1.Elasticsearch + } + tests := []struct { + name string + args args + want *APIKeyStore + wantErr bool + }{ + { + name: "Happy path", + args: args{ + c: k8s.NewFakeClient( + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{ "rc2" : { "namespace" : "ns2", "name" : "es2", "id": "SecretKeyID2" }, "rc1" : { "namespace" : "ns1", "name" : "es1", "id": "SecretKeyID1" } }`, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc1.credentials": []byte("SecretKeyValue1"), + "cluster.remote.rc2.credentials": []byte("SecretKeyValue2"), + }, + }, + ), + owner: es, + }, + want: &APIKeyStore{ + aliases: map[string]AliasValue{ + "rc1": {ID: "SecretKeyID1", Namespace: "ns1", Name: "es1"}, + "rc2": {ID: "SecretKeyID2", Namespace: "ns2", Name: "es2"}, + }, + keys: map[string]string{ + "rc1": "SecretKeyValue1", + "rc2": "SecretKeyValue2", + }, + }, + }, + { + name: "Secret does not exist", + args: args{ + c: k8s.NewFakeClient(), + owner: es, + }, + want: &APIKeyStore{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + got, err := LoadAPIKeyStore(ctx, tt.args.c, tt.args.owner) + if (err != nil) != tt.wantErr { + t.Errorf("LoadAPIKeyStore() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("LoadAPIKeyStore() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAPIKeyStore_Save(t *testing.T) { + type args struct { + c k8s.Client + } + tests := []struct { + name string + receiver *APIKeyStore + args args + wantErr bool + want *corev1.Secret + }{ + { + name: "Create a new store", + receiver: (&APIKeyStore{}). + Update("ns1", "es1", "rc1", "keyid1", "encodedValue1"). + Update("ns1", "es1", "rc2", "keyid2", "encodedValue2"), + args: args{c: k8s.NewFakeClient()}, + want: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + ResourceVersion: "1", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"rc1":{"namespace":"es1","name":"ns1","id":"keyid1"},"rc2":{"namespace":"es1","name":"ns1","id":"keyid2"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "myes", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "elasticsearch.k8s.elastic.co/v1", + Kind: "Elasticsearch", + Name: "myes", + UID: es.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc1.credentials": []byte("encodedValue1"), + "cluster.remote.rc2.credentials": []byte("encodedValue2"), + }, + }, + }, + { + name: "Delete the store", + receiver: (&APIKeyStore{}). + Update("ns1", "es1", "rc1", "keyid1", "encodedValue1"). + Update("ns2", "es2", "rc2", "keyid2", "encodedValue2"). + Delete("rc1").Delete("rc2"), + args: args{c: k8s.NewFakeClient(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + ResourceVersion: "1", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"rc1":"keyid1","rc2":"keyid2"}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "elasticsearch", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "myes", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "elasticsearch.k8s.elastic.co/v1", + Kind: "Elasticsearch", + Name: "myes", + UID: es.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc1.credentials": []byte("encodedValue1"), + "cluster.remote.rc2.credentials": []byte("encodedValue2"), + }, + })}, + }, + { + name: "Add new keys, remove another", + receiver: (&APIKeyStore{}). + Update("ns1", "es1", "rc1", "keyid1", "encodedValue1"). + Update("ns2", "es2", "rc2", "keyid2", "encodedValue2"). + Delete("rc1"). + Update("ns3", "es3", "rc3_1", "keyid3_1", "encodedValue31"). + Update("ns3", "es3", "rc3_2", "keyid3_2", "encodedValue32"), + args: args{c: k8s.NewFakeClient(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + ResourceVersion: "1", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"rc2":{"namespace":"es2","name":"ns2","id":"keyid2"},"rc1":{"namespace":"es1","name":"ns1","id":"keyid1"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "myes", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "elasticsearch.k8s.elastic.co/v1", + Kind: "Elasticsearch", + Name: "myes", + UID: es.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc1.credentials": []byte("encodedValue1"), + "unexpected_key": []byte("foo"), + "cluster.remote.rc2.credentials": []byte("encodedValue2"), + }, + })}, + want: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + ResourceVersion: "2", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"rc2":{"namespace":"es2","name":"ns2","id":"keyid2"},"rc3_1":{"namespace":"es3","name":"ns3","id":"keyid3_1"},"rc3_2":{"namespace":"es3","name":"ns3","id":"keyid3_2"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "myes", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "elasticsearch.k8s.elastic.co/v1", + Kind: "Elasticsearch", + Name: "myes", + UID: es.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc2.credentials": []byte("encodedValue2"), + "cluster.remote.rc3_1.credentials": []byte("encodedValue31"), + "cluster.remote.rc3_2.credentials": []byte("encodedValue32"), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + owner := es.DeepCopy() + if err := tt.receiver.Save(ctx, tt.args.c, owner); (err != nil) != tt.wantErr { + t.Errorf("APIKeyStore.Save() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.want != nil { + // get the Secret + apiKeysSecret := corev1.Secret{} + assert.NoError(t, tt.args.c.Get(ctx, types.NamespacedName{Name: "myes-es-remote-api-keys", Namespace: testNamespace}, &apiKeysSecret)) + if diff := cmp.Diff(*tt.want, apiKeysSecret); len(diff) > 0 { + t.Errorf("%s", diff) + } + } else { + // ensure the Secret does not exist + err := tt.args.c.Get(ctx, types.NamespacedName{Name: "myes-es-remote-api-keys", Namespace: testNamespace}, &corev1.Secret{}) + assert.Truef(t, errors.IsNotFound(err), "expected a 404 error") + } + }) + } +} diff --git a/pkg/controller/remoteca/labels.go b/pkg/controller/remotecluster/labels.go similarity index 98% rename from pkg/controller/remoteca/labels.go rename to pkg/controller/remotecluster/labels.go index edb2789c12..57bb5d3323 100644 --- a/pkg/controller/remoteca/labels.go +++ b/pkg/controller/remotecluster/labels.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "fmt" diff --git a/pkg/controller/remoteca/rbac.go b/pkg/controller/remotecluster/rbac.go similarity index 98% rename from pkg/controller/remoteca/rbac.go rename to pkg/controller/remotecluster/rbac.go index 991b129a8f..d5d92e9a9e 100644 --- a/pkg/controller/remoteca/rbac.go +++ b/pkg/controller/remotecluster/rbac.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "context" diff --git a/pkg/controller/remoteca/secret.go b/pkg/controller/remotecluster/secret.go similarity index 98% rename from pkg/controller/remoteca/secret.go rename to pkg/controller/remotecluster/secret.go index 6c09e518b0..605d2d1bc2 100644 --- a/pkg/controller/remoteca/secret.go +++ b/pkg/controller/remotecluster/secret.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "context" @@ -29,7 +29,7 @@ import ( // * Copy the CA of the remote cluster to the local one. func createOrUpdateCertificateAuthorities( ctx context.Context, - r *ReconcileRemoteCa, + r *ReconcileRemoteClusters, local, remote *esv1.Elasticsearch, ) *reconciler.Results { span, _ := apm.StartSpan(ctx, "create_or_update_remote_ca", tracing.SpanTypeApp) @@ -79,7 +79,7 @@ func createOrUpdateCertificateAuthorities( // copyCertificateAuthority creates a copy of the CA from a source cluster to a target cluster func copyCertificateAuthority( ctx context.Context, - r *ReconcileRemoteCa, + r *ReconcileRemoteClusters, source, target *esv1.Elasticsearch, ) error { sourceKey := k8s.ExtractNamespacedName(source) @@ -110,7 +110,7 @@ func copyCertificateAuthority( // remote cluster must be deleted from the local one. func deleteCertificateAuthorities( ctx context.Context, - r *ReconcileRemoteCa, + r *ReconcileRemoteClusters, local, remote types.NamespacedName, ) error { span, ctx := apm.StartSpan(ctx, "delete_certificate_authorities", tracing.SpanTypeApp) diff --git a/pkg/controller/remoteca/watches.go b/pkg/controller/remotecluster/watches.go similarity index 63% rename from pkg/controller/remoteca/watches.go rename to pkg/controller/remotecluster/watches.go index db5f852473..a7977601f0 100644 --- a/pkg/controller/remoteca/watches.go +++ b/pkg/controller/remotecluster/watches.go @@ -2,12 +2,14 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "context" "fmt" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -26,13 +28,34 @@ import ( ) // AddWatches set watches on objects needed to manage the association between a local and a remote cluster. -func addWatches(mgr manager.Manager, c controller.Controller, r *ReconcileRemoteCa) error { +func addWatches(mgr manager.Manager, c controller.Controller, r *ReconcileRemoteClusters) error { // Watch for changes to RemoteCluster if err := c.Watch(source.Kind(mgr.GetCache(), &esv1.Elasticsearch{}, &handler.TypedEnqueueRequestForObject[*esv1.Elasticsearch]{})); err != nil { return err } - // Watch Secrets that contain remote certificate authorities managed by this controller + // Emit changes to remote clusters to update API keys. + if err := c.Watch( + source.Kind( + mgr.GetCache(), + &esv1.Elasticsearch{}, + handler.TypedEnqueueRequestsFromMapFunc[*esv1.Elasticsearch, reconcile.Request]( + func(ctx context.Context, elasticsearch *esv1.Elasticsearch) []reconcile.Request { + requests := make([]reconcile.Request, 0, len(elasticsearch.Spec.RemoteClusters)) + for _, remoteCluster := range elasticsearch.Spec.RemoteClusters { + requests = append(requests, reconcile.Request{NamespacedName: remoteCluster.ElasticsearchRef.WithDefaultNamespace(elasticsearch.Namespace).NamespacedName()}) + } + return requests + }, + ), + ), + ); err != nil { + return err + } + + // Watch Secrets that contain: + // * Remote certificate authorities managed by this controller. + // * API keys if err := c.Watch( source.Kind(mgr.GetCache(), &v1.Secret{}, handler.TypedEnqueueRequestsFromMapFunc[*v1.Secret, reconcile.Request](newRequestsFromMatchedLabels()), @@ -58,21 +81,36 @@ func addWatches(mgr manager.Manager, c controller.Controller, r *ReconcileRemote // newRequestsFromMatchedLabels creates a watch handler function that creates reconcile requests based on the // labels set on a Secret which contains the remote CA. func newRequestsFromMatchedLabels() handler.TypedMapFunc[*v1.Secret, reconcile.Request] { - return handler.TypedMapFunc[*v1.Secret, reconcile.Request](func(ctx context.Context, obj *v1.Secret) []reconcile.Request { + return func(ctx context.Context, obj *v1.Secret) []reconcile.Request { labels := obj.GetLabels() - if !maps.ContainsKeys(labels, RemoteClusterNameLabelName, RemoteClusterNamespaceLabelName, commonv1.TypeLabelName) { - return nil + if maps.ContainsKeys(labels, RemoteClusterNameLabelName, RemoteClusterNamespaceLabelName, commonv1.TypeLabelName) { + // Remote cluster CA + if labels[commonv1.TypeLabelName] != remoteca.TypeLabelValue { + return nil + } + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Namespace: labels[RemoteClusterNamespaceLabelName], + Name: labels[RemoteClusterNameLabelName]}, + }, + } } - if labels[commonv1.TypeLabelName] != remoteca.TypeLabelValue { - return nil - } - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Namespace: labels[RemoteClusterNamespaceLabelName], - Name: labels[RemoteClusterNameLabelName]}, - }, + + if maps.ContainsKeys(labels, label.ClusterNameLabelName, commonv1.TypeLabelName) { + if labels[commonv1.TypeLabelName] != remoteClusterAPIKeysType { + return nil + } + // Remote cluster API keys Secret event. + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Namespace: obj.Namespace, + Name: labels[label.ClusterNameLabelName]}, + }, + } } - }) + + return nil + } } func watchName(local types.NamespacedName, remote types.NamespacedName) string { @@ -89,7 +127,7 @@ func watchName(local types.NamespacedName, remote types.NamespacedName) string { // The local CA is watched to update the trusted certificates in the remote clusters. // The remote CAs are watched to update the trusted certificates of the local cluster. func addCertificatesAuthorityWatches( - reconcileClusterAssociation *ReconcileRemoteCa, + reconcileClusterAssociation *ReconcileRemoteClusters, local, remote types.NamespacedName) error { // Watch the CA secret of Elasticsearch clusters which are involved in a association. err := reconcileClusterAssociation.watches.Secrets.AddHandler(watches.NamedWatch[*corev1.Secret]{ diff --git a/pkg/telemetry/fixtures.go b/pkg/telemetry/fixtures.go index 0b817787c3..b5087f719e 100644 --- a/pkg/telemetry/fixtures.go +++ b/pkg/telemetry/fixtures.go @@ -21,7 +21,8 @@ type ElasticsearchTemplateData struct { StackMonitoringLogsCount int32 StackMonitoringMetricsCount int32 PodCount int32 - + RemoteClustersCount int32 + RemoteClustersAPIKeysCount int32 *NodeLabelsTemplateData } @@ -79,6 +80,8 @@ const expectedTelemetryTemplate = `eck: {{- end }} helm_resource_count: 0 pod_count: {{ .ElasticsearchTemplateData.PodCount }} + remote_clusters_api_keys_count: {{ .ElasticsearchTemplateData.RemoteClustersAPIKeysCount }} + remote_clusters_count: {{ .ElasticsearchTemplateData.RemoteClustersCount }} resource_count: {{ .ElasticsearchTemplateData.ResourceCount }} stack_monitoring_logs_count: {{ .ElasticsearchTemplateData.StackMonitoringLogsCount }} stack_monitoring_metrics_count: {{ .ElasticsearchTemplateData.StackMonitoringMetricsCount }} diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index 70f726518d..660c1e91f2 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -232,6 +232,8 @@ func esStats(k8sClient k8s.Client, managedNamespaces []string) (string, interfac AutoscaledResourceCount int32 `json:"autoscaled_resource_count"` StackMonitoringLogsCount int32 `json:"stack_monitoring_logs_count"` StackMonitoringMetricsCount int32 `json:"stack_monitoring_metrics_count"` + RemoteClustersCount int32 `json:"remote_clusters_count"` + RemoteClustersAPIKeysCount int32 `json:"remote_clusters_api_keys_count"` DownwardNodeLabels *downwardNodeLabelsStats `json:"downward_node_labels,omitempty"` }{} distinctNodeLabels := set.Make() @@ -247,6 +249,10 @@ func esStats(k8sClient k8s.Client, managedNamespaces []string) (string, interfac stats.ResourceCount++ stats.PodCount += es.Status.AvailableNodes + rcWithoutAPIKeys, rcWithAPIKeys := es.RemoteClustersCount() + stats.RemoteClustersCount += rcWithoutAPIKeys + stats.RemoteClustersAPIKeysCount += rcWithAPIKeys + if isManagedByHelm(es.Labels) { stats.HelmManagedResourceCount++ } diff --git a/pkg/telemetry/telemetry_test.go b/pkg/telemetry/telemetry_test.go index 7c54ab149b..08a29651fa 100644 --- a/pkg/telemetry/telemetry_test.go +++ b/pkg/telemetry/telemetry_test.go @@ -471,6 +471,8 @@ func TestNewReporter(t *testing.T) { autoscaled_resource_count: 2 helm_resource_count: 1 pod_count: 10 + remote_clusters_api_keys_count: 0 + remote_clusters_count: 0 resource_count: 4 stack_monitoring_logs_count: 1 stack_monitoring_metrics_count: 1 @@ -597,6 +599,8 @@ func TestReporter_report(t *testing.T) { PodCount: 9, ResourceCount: 3, StackMonitoringMetricsCount: 2, + RemoteClustersCount: 0, + RemoteClustersAPIKeysCount: 0, }, }, }, @@ -631,9 +635,11 @@ func TestReporter_report(t *testing.T) { }, wantData: TemplateData{ ElasticsearchTemplateData{ - PodCount: 10, - ResourceCount: 2, - StackMonitoringLogsCount: 1, + PodCount: 10, + ResourceCount: 2, + StackMonitoringLogsCount: 1, + RemoteClustersCount: 0, + RemoteClustersAPIKeysCount: 0, }, }, }, { @@ -665,8 +671,10 @@ func TestReporter_report(t *testing.T) { }, wantData: TemplateData{ ElasticsearchTemplateData{ - PodCount: 4, - ResourceCount: 2, + PodCount: 4, + ResourceCount: 2, + RemoteClustersCount: 0, + RemoteClustersAPIKeysCount: 0, NodeLabelsTemplateData: &NodeLabelsTemplateData{ ResourceWithNodeLabelsCount: 1, DistinctNodeLabelsCount: 1, @@ -705,8 +713,10 @@ func TestReporter_report(t *testing.T) { }, wantData: TemplateData{ ElasticsearchTemplateData{ - PodCount: 4, - ResourceCount: 2, + PodCount: 4, + ResourceCount: 2, + RemoteClustersCount: 0, + RemoteClustersAPIKeysCount: 0, NodeLabelsTemplateData: &NodeLabelsTemplateData{ ResourceWithNodeLabelsCount: 2, DistinctNodeLabelsCount: 3, // ns/label1,ns/label2 and ns/label3 @@ -714,6 +724,67 @@ func TestReporter_report(t *testing.T) { }, }, }, + { + name: "With remote clusters", + fields: fields{ + objects: []client.Object{ + &esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: "es1", + }, + Spec: esv1.ElasticsearchSpec{ + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "rc1", + APIKey: nil, + }, + { + Name: "rc2", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + }, + }, + Status: esv1.ElasticsearchStatus{ + AvailableNodes: 2, + }, + }, + &esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: "es2", + }, + Spec: esv1.ElasticsearchSpec{ + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "rc1", + APIKey: nil, + }, + { + Name: "rc2", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + { + Name: "rc3", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + }, + }, + Status: esv1.ElasticsearchStatus{ + AvailableNodes: 8, + }, + }, + }, + }, + wantData: TemplateData{ + ElasticsearchTemplateData{ + PodCount: 10, + ResourceCount: 2, + RemoteClustersCount: 1 + 1, + RemoteClustersAPIKeysCount: 1 + 2, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From fb34ee7a2d06684d3f1d178f2939b508cbe07f93 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Wed, 9 Oct 2024 10:32:04 +0200 Subject: [PATCH 02/21] Cosmetic changes --- pkg/controller/elasticsearch/driver/driver.go | 21 ++-- pkg/controller/remotecluster/apikey.go | 110 ++++++++++++------ pkg/utils/k8s/k8sutils.go | 19 +++ 3 files changed, 99 insertions(+), 51 deletions(-) diff --git a/pkg/controller/elasticsearch/driver/driver.go b/pkg/controller/elasticsearch/driver/driver.go index 3a216ff362..c113ab8c78 100644 --- a/pkg/controller/elasticsearch/driver/driver.go +++ b/pkg/controller/elasticsearch/driver/driver.go @@ -11,6 +11,8 @@ import ( "strings" "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -162,20 +164,13 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { } } else { // Ensure that remote cluster Service does not exist. - remoteClusterService := &corev1.Service{} - remoteClusterServiceName := types.NamespacedName{ - Name: services.RemoteClusterServiceName(d.ES.Name), - Namespace: d.ES.Namespace, - } - if err := d.Client.Get(ctx, remoteClusterServiceName, remoteClusterService); err != nil { - if !k8serrors.IsNotFound(err) { - results.WithError(err) - } - } else { - // Remote cluster Service has been found but is not expected. - log.Info("Deleting remote cluster Service") - results.WithError(d.Client.Delete(ctx, remoteClusterService)) + remoteClusterService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: d.ES.Namespace, + Name: services.RemoteClusterServiceName(d.ES.Name), + }, } + results.WithError(k8s.DeleteResourceIfExists(ctx, d.Client, remoteClusterService)) } resourcesState, err := reconcile.NewResourcesStateFromAPI(d.Client, d.ES) diff --git a/pkg/controller/remotecluster/apikey.go b/pkg/controller/remotecluster/apikey.go index b27e82e6dd..75b595c73e 100644 --- a/pkg/controller/remotecluster/apikey.go +++ b/pkg/controller/remotecluster/apikey.go @@ -8,6 +8,7 @@ import ( "context" "fmt" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" @@ -67,46 +68,13 @@ func reconcileAPIKeys( activeAPIKey := activeAPIKeys.GetActiveKeyWithName(apiKeyName) expectedHash := hash.HashObject(remoteCluster.APIKey) if activeAPIKey == nil { - // Active API key not found, let's create a new one. - log.Info("Creating API key", "alias", remoteCluster.Name, "key", apiKeyName) - apiKey, err := esClient.CreateCrossClusterAPIKey(ctx, esclient.CrossClusterAPIKeyCreateRequest{ - Name: apiKeyName, - CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ - RemoteClusterAPIKey: *remoteCluster.APIKey, - Metadata: newMetadataFor(clientES, expectedHash), - }, - }) - if err != nil { + if err := createAPIKey(ctx, log, remoteCluster, apiKeyName, esClient, clientES, expectedHash, clientClusterAPIKeyStore, reconciledES); err != nil { return err } - clientClusterAPIKeyStore.Update(reconciledES.Name, reconciledES.Namespace, remoteCluster.Name, apiKey.ID, apiKey.Encoded) - } - // If an API key already exists ensure that the access field is the expected one using the hash - if activeAPIKey != nil { - // Ensure that the API key is in the keystore - if clientClusterAPIKeyStore.KeyIDFor(remoteCluster.Name) != activeAPIKey.ID { - // We have a problem here, the API Key ID in Elasticsearch does not match the API Key recorded in the Secret. - // Invalidate the API Key in ES and requeue - log.Info("Invalidating API key as it does not match the one in keystore", "alias", remoteCluster.Name, "key", apiKeyName) - if err := esClient.InvalidateCrossClusterAPIKey(ctx, activeAPIKey.Name); err != nil { - return err - } - return fmt.Errorf( - "cluster key id for alias %s %s (%s), does not match the one stored in the keystore of %s/%s", - remoteCluster.Name, activeAPIKey.Name, activeAPIKey.ID, clientES.Namespace, clientES.Name, - ) - } - currentHash := activeAPIKey.Metadata["elasticsearch.k8s.elastic.co/config-hash"] - if currentHash != expectedHash { - log.Info("Updating API key", "alias", remoteCluster.Name) - // Update the Key - _, err := esClient.UpdateCrossClusterAPIKey(ctx, activeAPIKey.ID, esclient.CrossClusterAPIKeyUpdateRequest{ - RemoteClusterAPIKey: *remoteCluster.APIKey, - Metadata: newMetadataFor(clientES, expectedHash), - }) - if err != nil { - return err - } + } else { + // If an API key already exists ensure that the access field is the expected one using the hash + if err := maybeUpdateAPIKey(ctx, log, esClient, clientClusterAPIKeyStore, remoteCluster, activeAPIKey, apiKeyName, clientES, expectedHash); err != nil { + return err } } } @@ -143,6 +111,72 @@ func reconcileAPIKeys( return nil } +func createAPIKey( + ctx context.Context, + log logr.Logger, + remoteCluster esv1.RemoteCluster, + apiKeyName string, + esClient esclient.Client, + clientES *esv1.Elasticsearch, + expectedHash string, + clientClusterAPIKeyStore *APIKeyStore, + reconciledES *esv1.Elasticsearch, +) error { + // Active API key not found, let's create a new one. + log.Info("Creating API key", "alias", remoteCluster.Name, "key", apiKeyName) + apiKey, err := esClient.CreateCrossClusterAPIKey(ctx, esclient.CrossClusterAPIKeyCreateRequest{ + Name: apiKeyName, + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + RemoteClusterAPIKey: *remoteCluster.APIKey, + Metadata: newMetadataFor(clientES, expectedHash), + }, + }) + if err != nil { + return err + } + clientClusterAPIKeyStore.Update(reconciledES.Name, reconciledES.Namespace, remoteCluster.Name, apiKey.ID, apiKey.Encoded) + return nil +} + +func maybeUpdateAPIKey( + ctx context.Context, + log logr.Logger, + esClient esclient.Client, + clientClusterAPIKeyStore *APIKeyStore, + remoteCluster esv1.RemoteCluster, + activeAPIKey *esclient.CrossClusterAPIKey, + apiKeyName string, + clientES *esv1.Elasticsearch, + expectedHash string, +) error { + // Ensure that the API key is in the keystore + if clientClusterAPIKeyStore.KeyIDFor(remoteCluster.Name) != activeAPIKey.ID { + // We have a problem here, the API Key ID in Elasticsearch does not match the API Key recorded in the Secret. + // Invalidate the API Key in ES and requeue + log.Info("Invalidating API key as it does not match the one in keystore", "alias", remoteCluster.Name, "key", apiKeyName) + if err := esClient.InvalidateCrossClusterAPIKey(ctx, activeAPIKey.Name); err != nil { + return err + } + return fmt.Errorf( + "cluster key id for alias %s %s (%s), does not match the one stored in the keystore of %s/%s", + remoteCluster.Name, activeAPIKey.Name, activeAPIKey.ID, clientES.Namespace, clientES.Name, + ) + } + currentHash := activeAPIKey.Metadata["elasticsearch.k8s.elastic.co/config-hash"] + if currentHash != expectedHash { + log.Info("Updating API key", "alias", remoteCluster.Name) + // Update the Key + _, err := esClient.UpdateCrossClusterAPIKey(ctx, activeAPIKey.ID, esclient.CrossClusterAPIKeyUpdateRequest{ + RemoteClusterAPIKey: *remoteCluster.APIKey, + Metadata: newMetadataFor(clientES, expectedHash), + }) + if err != nil { + return err + } + } + return nil +} + // newMetadataFor returns the metadata to be set in the Elasticsearch API keys metadata in the Elasticsearch cluster // state, not on a Kubernetes object. func newMetadataFor(clientES *esv1.Elasticsearch, expectedHash string) map[string]interface{} { diff --git a/pkg/utils/k8s/k8sutils.go b/pkg/utils/k8s/k8sutils.go index a68c16d769..a676052b20 100644 --- a/pkg/utils/k8s/k8sutils.go +++ b/pkg/utils/k8s/k8sutils.go @@ -226,6 +226,25 @@ func DeleteSecretIfExists(ctx context.Context, c Client, key types.NamespacedNam return err } +// DeleteResourceIfExists deletes the provided resource if exists. +func DeleteResourceIfExists(ctx context.Context, c Client, obj client.Object) error { + key := types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + if err := c.Get(ctx, key, obj); err != nil { + if apierrors.IsNotFound(err) { + // Resource does not exist. + return nil + } + return err + } + if err := c.Delete(ctx, obj); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + // PodsMatchingLabels returns Pods from the given namespace matching the given labels. func PodsMatchingLabels(c Client, namespace string, labels map[string]string) ([]corev1.Pod, error) { var pods corev1.PodList From ed29168543d20b73b318bfdab5e7309d21575457 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Thu, 10 Oct 2024 08:10:35 +0200 Subject: [PATCH 03/21] Publish the remote cluster Service to the client, not the Pod IP --- pkg/apis/elasticsearch/v1/fields.go | 4 +++- pkg/controller/elasticsearch/nodespec/podspec.go | 8 ++++++++ pkg/controller/elasticsearch/settings/environment.go | 1 + .../elasticsearch/settings/merged_config.go | 11 ++++++++++- .../elasticsearch/settings/merged_config_test.go | 4 ++++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pkg/apis/elasticsearch/v1/fields.go b/pkg/apis/elasticsearch/v1/fields.go index fd7a9daa97..8ea5c4bf6b 100644 --- a/pkg/apis/elasticsearch/v1/fields.go +++ b/pkg/apis/elasticsearch/v1/fields.go @@ -26,7 +26,9 @@ const ( NetworkPublishHost = "network.publish_host" HTTPPublishHost = "http.publish_host" - RemoteClusterEnabled = "remote_cluster_server.enabled" + RemoteClusterEnabled = "remote_cluster_server.enabled" + RemoteClusterPublishHost = "remote_cluster.publish_host" + RemoteClusterBindHost = "remote_cluster.bind_host" NodeName = "node.name" diff --git a/pkg/controller/elasticsearch/nodespec/podspec.go b/pkg/controller/elasticsearch/nodespec/podspec.go index 03d01eb500..bb3c958aed 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec.go +++ b/pkg/controller/elasticsearch/nodespec/podspec.go @@ -27,6 +27,7 @@ import ( "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/network" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/securitycontext" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/settings" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/stackmon" esvolume "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/volume" @@ -134,6 +135,13 @@ func BuildPodTemplateSpec( WithContainersSecurityContext(securitycontext.For(ver, enableReadOnlyRootFilesystem)). WithPreStopHook(*NewPreStopHook()) + if es.Spec.RemoteClusterServer.Enabled { + builder = builder.WithEnv(corev1.EnvVar{ + Name: settings.EnvRemoteClusterService, + Value: services.RemoteClusterServiceName(es.Name), + }) + } + builder, err = stackmon.WithMonitoring(ctx, client, builder, es) if err != nil { return corev1.PodTemplateSpec{}, err diff --git a/pkg/controller/elasticsearch/settings/environment.go b/pkg/controller/elasticsearch/settings/environment.go index aa10548f10..5490880863 100644 --- a/pkg/controller/elasticsearch/settings/environment.go +++ b/pkg/controller/elasticsearch/settings/environment.go @@ -12,6 +12,7 @@ const ( EnvProbeUsername = "PROBE_USERNAME" EnvReadinessProbeProtocol = "READINESS_PROBE_PROTOCOL" HeadlessServiceName = "HEADLESS_SERVICE_NAME" + EnvRemoteClusterService = "REMOTE_CLUSTER_SERVICE" // These are injected as env var into the ES pod at runtime, // to be referenced in ES configuration file diff --git a/pkg/controller/elasticsearch/settings/merged_config.go b/pkg/controller/elasticsearch/settings/merged_config.go index 5e6dbfe682..5e6723ec12 100644 --- a/pkg/controller/elasticsearch/settings/merged_config.go +++ b/pkg/controller/elasticsearch/settings/merged_config.go @@ -75,6 +75,11 @@ func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamil if remoteClusterServerEnabled { cfg[esv1.RemoteClusterEnabled] = "true" + // This is required when each Pod server certificates do not contain the Pod's IP address, which might be the case + // if the certificates are generated by the cert-manager CSI driver for example. Using this parameter the remote client + // cluster is going to try to connect to the remote cluster service using the Service and each specific Pod IP. + cfg[esv1.RemoteClusterPublishHost] = "${" + EnvRemoteClusterService + "}.${" + EnvNamespace + "}.svc" + cfg[esv1.RemoteClusterBindHost] = "0.0.0.0" } // seed hosts setting name changed starting ES 7.X @@ -143,7 +148,11 @@ func xpackConfig(ver version.Version, httpCfg commonv1.HTTPConfig, remoteCluster if remoteClusterClientEnabled { cfg[esv1.XPackSecurityRemoteClusterClientSslKey] = true - cfg[esv1.XPackSecurityRemoteClusterClientSslCertificateAuthorities] = path.Join(volume.RemoteCertificateAuthoritiesSecretVolumeMountPath, certificates.CAFileName) + cfg[esv1.XPackSecurityRemoteClusterClientSslCertificateAuthorities] = []string{ + // Include /usr/share/elasticsearch/config/transport-certs/ca.crt to trust any additional CA in transport.tls.certificateAuthorities + path.Join(volume.TransportCertificatesSecretVolumeMountPath, certificates.CAFileName), + path.Join(volume.RemoteCertificateAuthoritiesSecretVolumeMountPath, certificates.CAFileName), + } } // always enable the built-in file and native internal realms for user auth, ordered as first diff --git a/pkg/controller/elasticsearch/settings/merged_config_test.go b/pkg/controller/elasticsearch/settings/merged_config_test.go index e932c961cc..b14ed69b32 100644 --- a/pkg/controller/elasticsearch/settings/merged_config_test.go +++ b/pkg/controller/elasticsearch/settings/merged_config_test.go @@ -93,6 +93,10 @@ func TestNewMergedESConfig(t *testing.T) { require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) // Remote cluster server configuration. + require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster_server.enabled"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster.publish_host"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster.bind_host"}))) + // Remote cluster server TLS configuration. require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.key"}))) require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate"}))) require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate_authorities"}))) From 4c2c6fe976d173f5c1f64e395512ee4b59aa02c8 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Thu, 10 Oct 2024 09:52:35 +0200 Subject: [PATCH 04/21] Use existing headless service --- pkg/apis/elasticsearch/v1/fields.go | 2 +- .../certificates/transport/csr.go | 3 +++ .../elasticsearch/nodespec/podspec.go | 8 ------ .../elasticsearch/nodespec/statefulset.go | 25 +++++++++++++------ .../elasticsearch/services/services.go | 25 +++++++++++-------- .../elasticsearch/settings/environment.go | 1 - .../elasticsearch/settings/merged_config.go | 7 ++---- .../settings/merged_config_test.go | 2 +- 8 files changed, 40 insertions(+), 33 deletions(-) diff --git a/pkg/apis/elasticsearch/v1/fields.go b/pkg/apis/elasticsearch/v1/fields.go index 8ea5c4bf6b..033a86c86d 100644 --- a/pkg/apis/elasticsearch/v1/fields.go +++ b/pkg/apis/elasticsearch/v1/fields.go @@ -28,7 +28,7 @@ const ( RemoteClusterEnabled = "remote_cluster_server.enabled" RemoteClusterPublishHost = "remote_cluster.publish_host" - RemoteClusterBindHost = "remote_cluster.bind_host" + RemoteClusterHost = "remote_cluster.host" NodeName = "node.name" diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index 9993d0499e..8a71c5b381 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -109,7 +109,10 @@ func buildGeneralNames( // since these are the ones also used in the context of remote clusters access using API keys. generalNames = append( generalNames, + // Remote cluster headless service certificates.GeneralName{DNSName: fmt.Sprintf("%s.%s.svc", esv1.RemoteClusterService(cluster.Name), cluster.Namespace)}, + // Individual remote_cluster.publish_host is set to ...svc + certificates.GeneralName{DNSName: fmt.Sprintf("%s.%s.%s.svc", pod.Name, svcName, cluster.Namespace)}, ) } diff --git a/pkg/controller/elasticsearch/nodespec/podspec.go b/pkg/controller/elasticsearch/nodespec/podspec.go index bb3c958aed..03d01eb500 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec.go +++ b/pkg/controller/elasticsearch/nodespec/podspec.go @@ -27,7 +27,6 @@ import ( "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/network" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/securitycontext" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/settings" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/stackmon" esvolume "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/volume" @@ -135,13 +134,6 @@ func BuildPodTemplateSpec( WithContainersSecurityContext(securitycontext.For(ver, enableReadOnlyRootFilesystem)). WithPreStopHook(*NewPreStopHook()) - if es.Spec.RemoteClusterServer.Enabled { - builder = builder.WithEnv(corev1.EnvVar{ - Name: settings.EnvRemoteClusterService, - Value: services.RemoteClusterServiceName(es.Name), - }) - } - builder, err = stackmon.WithMonitoring(ctx, client, builder, es) if err != nil { return corev1.PodTemplateSpec{}, err diff --git a/pkg/controller/elasticsearch/nodespec/statefulset.go b/pkg/controller/elasticsearch/nodespec/statefulset.go index 6d9d0eda9f..9a5cca01f2 100644 --- a/pkg/controller/elasticsearch/nodespec/statefulset.go +++ b/pkg/controller/elasticsearch/nodespec/statefulset.go @@ -18,6 +18,7 @@ import ( sset "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/statefulset" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/network" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/settings" es_sset "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/sset" esvolume "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/volume" @@ -33,6 +34,22 @@ func HeadlessServiceName(ssetName string) string { // HeadlessService returns a headless service for the given StatefulSet func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { nsn := k8s.ExtractNamespacedName(es) + ports := []corev1.ServicePort{ + { + Name: es.Spec.HTTP.Protocol(), + Protocol: corev1.ProtocolTCP, + Port: network.HTTPPort, + }, + } + if es.Spec.RemoteClusterServer.Enabled { + ports = append(ports, + corev1.ServicePort{ + Name: services.RemoteClusterServicePortName, + Protocol: corev1.ProtocolTCP, + Port: network.RemoteClusterPort, + }, + ) + } return corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -44,13 +61,7 @@ func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { Type: corev1.ServiceTypeClusterIP, ClusterIP: corev1.ClusterIPNone, Selector: label.NewStatefulSetLabels(nsn, ssetName), - Ports: []corev1.ServicePort{ - { - Name: es.Spec.HTTP.Protocol(), - Protocol: corev1.ProtocolTCP, - Port: network.HTTPPort, - }, - }, + Ports: ports, // allow nodes to discover themselves via DNS while they are booting up ie. are not ready yet PublishNotReadyAddresses: true, }, diff --git a/pkg/controller/elasticsearch/services/services.go b/pkg/controller/elasticsearch/services/services.go index b2b23d18a7..c3fb559b27 100644 --- a/pkg/controller/elasticsearch/services/services.go +++ b/pkg/controller/elasticsearch/services/services.go @@ -24,6 +24,8 @@ import ( const ( globalServiceSuffix = ".svc" + + RemoteClusterServicePortName = "rcs" ) // TransportServiceName returns the name for the transport service associated to this cluster @@ -153,25 +155,28 @@ func NewInternalService(es esv1.Elasticsearch) *corev1.Service { // NewRemoteClusterService returns the service associated to the remote cluster service for the given cluster. func NewRemoteClusterService(es esv1.Elasticsearch) *corev1.Service { - return &corev1.Service{ + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: RemoteClusterServiceName(es.Name), Namespace: es.Namespace, Labels: label.NewLabels(k8s.ExtractNamespacedName(&es)), }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{ - { - Name: "rcs", - Protocol: corev1.ProtocolTCP, - Port: network.RemoteClusterPort, - }, - }, + Type: corev1.ServiceTypeClusterIP, Selector: label.NewLabels(k8s.ExtractNamespacedName(&es)), - PublishNotReadyAddresses: false, + PublishNotReadyAddresses: true, + ClusterIP: "None", + }, + } + labels := label.NewLabels(k8s.ExtractNamespacedName(&es)) + ports := []corev1.ServicePort{ + { + Name: RemoteClusterServicePortName, + Protocol: corev1.ProtocolTCP, + Port: network.RemoteClusterPort, }, } + return defaults.SetServiceDefaults(svc, labels, labels, ports) } type urlProvider struct { diff --git a/pkg/controller/elasticsearch/settings/environment.go b/pkg/controller/elasticsearch/settings/environment.go index 5490880863..aa10548f10 100644 --- a/pkg/controller/elasticsearch/settings/environment.go +++ b/pkg/controller/elasticsearch/settings/environment.go @@ -12,7 +12,6 @@ const ( EnvProbeUsername = "PROBE_USERNAME" EnvReadinessProbeProtocol = "READINESS_PROBE_PROTOCOL" HeadlessServiceName = "HEADLESS_SERVICE_NAME" - EnvRemoteClusterService = "REMOTE_CLUSTER_SERVICE" // These are injected as env var into the ES pod at runtime, // to be referenced in ES configuration file diff --git a/pkg/controller/elasticsearch/settings/merged_config.go b/pkg/controller/elasticsearch/settings/merged_config.go index 5e6723ec12..067ec67a3e 100644 --- a/pkg/controller/elasticsearch/settings/merged_config.go +++ b/pkg/controller/elasticsearch/settings/merged_config.go @@ -75,11 +75,8 @@ func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamil if remoteClusterServerEnabled { cfg[esv1.RemoteClusterEnabled] = "true" - // This is required when each Pod server certificates do not contain the Pod's IP address, which might be the case - // if the certificates are generated by the cert-manager CSI driver for example. Using this parameter the remote client - // cluster is going to try to connect to the remote cluster service using the Service and each specific Pod IP. - cfg[esv1.RemoteClusterPublishHost] = "${" + EnvRemoteClusterService + "}.${" + EnvNamespace + "}.svc" - cfg[esv1.RemoteClusterBindHost] = "0.0.0.0" + cfg[esv1.RemoteClusterPublishHost] = "${" + EnvPodName + "}.${" + HeadlessServiceName + "}.${" + EnvNamespace + "}.svc" + cfg[esv1.RemoteClusterHost] = "0" } // seed hosts setting name changed starting ES 7.X diff --git a/pkg/controller/elasticsearch/settings/merged_config_test.go b/pkg/controller/elasticsearch/settings/merged_config_test.go index b14ed69b32..7651650e4b 100644 --- a/pkg/controller/elasticsearch/settings/merged_config_test.go +++ b/pkg/controller/elasticsearch/settings/merged_config_test.go @@ -95,7 +95,7 @@ func TestNewMergedESConfig(t *testing.T) { // Remote cluster server configuration. require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster_server.enabled"}))) require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster.publish_host"}))) - require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster.bind_host"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster.host"}))) // Remote cluster server TLS configuration. require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.key"}))) require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate"}))) From 87355924d08bdf740ca55b8c9ebb25193324521a Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Fri, 11 Oct 2024 08:22:52 +0200 Subject: [PATCH 05/21] [DOC] Update issuing node transport certificates with third-party tools --- .../elasticsearch/transport-settings.asciidoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc b/docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc index 67b802e6ba..766876c9ba 100644 --- a/docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc @@ -115,12 +115,16 @@ spec: volumeAttributes: csi.cert-manager.io/issuer-name: ca-cluster-issuer <2> csi.cert-manager.io/issuer-kind: ClusterIssuer - csi.cert-manager.io/dns-names: "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local" + csi.cert-manager.io/dns-names: "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local" <3> ---- <1> Disables the self-signed certificates generated by ECK for the transport layer. <2> The example assumes that a `ClusterIssuer` by the name of `ca-cluster-issuer` exists and a PEM encoded version of the CA certificate is available in a ConfigMap (in the example named `trust`). The CA certificate must be in a file called `ca.crt` inside the ConfigMap in the same namespace as the Elasticsearch resource. +<3> If the remote cluster server is enabled, then the DNS names must also include both: +* The DNS name for the related Kubernetes `Service`: `-es-remote-cluster.${POD_NAMESPACE}.svc` +* The Pod DNS name: `${POD_NAME}.-es-.${POD_NAMESPACE}.svc` + The following manifest is only provided to illustrate how these certificates can be configured in principle, using the trust-manager Bundle resource and cert-manager provisioned certificates: [source,yaml,subs="attributes,callouts"] From 5ec1e8d30b36fbcfd7141eb5aa37c012639d2755 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Fri, 11 Oct 2024 10:52:52 +0200 Subject: [PATCH 06/21] [E2E] Add end-to-end test --- test/e2e/es/remote_cluster_test.go | 150 ++++++++++++++++++++++++- test/e2e/test/elasticsearch/builder.go | 16 +++ 2 files changed, 165 insertions(+), 1 deletion(-) diff --git a/test/e2e/es/remote_cluster_test.go b/test/e2e/es/remote_cluster_test.go index ecbaf11237..6ecd58d3a3 100644 --- a/test/e2e/es/remote_cluster_test.go +++ b/test/e2e/es/remote_cluster_test.go @@ -12,10 +12,13 @@ import ( "fmt" "net/http" "os" + "strings" "testing" "github.com/stretchr/testify/require" + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" "github.com/elastic/cloud-on-k8s/v2/test/e2e/test" "github.com/elastic/cloud-on-k8s/v2/test/e2e/test/elasticsearch" @@ -28,7 +31,7 @@ const ( // TestRemoteCluster tests the local K8S remote cluster feature. // 1. In a first Elasticsearch cluster some data are indexed in the "data-integrity-check" index. // 2. A second cluster is created with the first cluster declared as a remote cluster. -// 3. Finally an index follower is created in the second cluster to follow the "data-integrity-check" index from the first one. +// 3. Finally, an index follower is created in the second cluster to follow the "data-integrity-check" index from the first one. func TestRemoteCluster(t *testing.T) { // only execute this test if we have a test license to work with if test.Ctx().TestLicense == "" { @@ -85,6 +88,10 @@ func TestRemoteCluster(t *testing.T) { require.NoError(t, elasticsearch.NewDataIntegrityCheck(k, es1Builder).WithSoftDeletesEnabled(true).Init()) }, }, + test.Step{ + Name: "Check that remote cluster is connected using transport protocol", + Test: test.Eventually(checkRemoteClusterSeed(k, es2Builder, es1Builder, 9300)), + }, test.Step{ Name: "Create a follower index on the second cluster", Test: func(t *testing.T) { @@ -114,3 +121,144 @@ func TestRemoteCluster(t *testing.T) { test.Sequence(before, stepsFn, es1Builder, es2Builder).RunSequential(t) } + +// TestRemoteCluster tests the K8S remote cluster feature using API Keys. +// This test is similar to TestRemoteCluster: +// 1. In a first Elasticsearch cluster some data are indexed in the "data-integrity-check" index. +// 2. A second cluster is created with the first cluster declared as a remote cluster. +// 3. Finally, an index follower is created in the second cluster to follow the "data-integrity-check" index from the first one. +func TestRemoteClusterWithAPIKeys(t *testing.T) { + // only execute this test if we have a test license to work with + if test.Ctx().TestLicense == "" { + t.SkipNow() + } + + // only execute the test if Elasticsearch supports API keys + if version.MustParse(test.Ctx().ElasticStackVersion).LT(esv1.RemoteClusterAPIKeysMinVersion) { + t.Skipf("%s does not support remote cluster API keys", esv1.RemoteClusterAPIKeysMinVersion) + } + + name := "test-remote-cluster-api-keys" + ns1 := test.Ctx().ManagedNamespace(0) + es1Builder := elasticsearch.NewBuilder(name). + WithNamespace(ns1). + WithESMasterDataNodes(1, elasticsearch.DefaultResources). + WithRemoteClusterServer(). + WithRestrictedSecurityContext() + es1LicenseTestContext := elasticsearch.NewLicenseTestContext(test.NewK8sClientOrFatal(), es1Builder.Elasticsearch) + + ns2 := test.Ctx().ManagedNamespace(1) + es2Builder := elasticsearch.NewBuilder(name). + WithNamespace(ns2). + WithESMasterDataNodes(1, elasticsearch.DefaultResources). + WithRestrictedSecurityContext(). + WithRemoteClusterAPIKey(es1Builder, &esv1.RemoteClusterAPIKey{ + Access: esv1.RemoteClusterAccess{ + Search: &esv1.Search{ + Names: []string{elasticsearch.DataIntegrityIndex}, + }, + Replication: &esv1.Replication{ + Names: []string{elasticsearch.DataIntegrityIndex}, + }, + }, + }) + es2LicenseTestContext := elasticsearch.NewLicenseTestContext(test.NewK8sClientOrFatal(), es2Builder.Elasticsearch) + licenseBytes, err := os.ReadFile(test.Ctx().TestLicense) + require.NoError(t, err) + trialSecretName := "eck-license" + + before := func(k *test.K8sClient) test.StepList { + // Deploy a Trial license + return test.StepList{ + es1LicenseTestContext.DeleteAllEnterpriseLicenseSecrets(), + es1LicenseTestContext.CreateEnterpriseLicenseSecret(trialSecretName, licenseBytes), + } + } + + followerIndex := "data-integrity-check-follower" + stepsFn := func(k *test.K8sClient) test.StepList { + return test.StepList{ + // Init license test context + es1LicenseTestContext.Init(), + es2LicenseTestContext.Init(), + // Check that the first cluster is using a Platinum license + es1LicenseTestContext.CheckElasticsearchLicense( + client.ElasticsearchLicenseTypePlatinum, + client.ElasticsearchLicenseTypeEnterprise, + ), + // Check that the second cluster is using a Platinum license + es1LicenseTestContext.CheckElasticsearchLicense( + client.ElasticsearchLicenseTypePlatinum, + client.ElasticsearchLicenseTypeEnterprise, + ), + test.Step{ + Name: "Add some data to the first cluster", + Test: func(t *testing.T) { + // Always enable soft deletes on test index. This is required to create follower indices but disabled by default on 6.x + require.NoError(t, elasticsearch.NewDataIntegrityCheck(k, es1Builder).WithSoftDeletesEnabled(true).Init()) + }, + }, + test.Step{ + Name: "Check that remote cluster is connected to remote cluster server", + Test: test.Eventually(checkRemoteClusterSeed(k, es2Builder, es1Builder, 9443)), + }, + test.Step{ + Name: "Create a follower index on the second cluster", + Test: func(t *testing.T) { + esClient, err := elasticsearch.NewElasticsearchClient(es2Builder.Elasticsearch, k) + require.NoError(t, err) + // create the index with controlled settings + followerCreation, err := http.NewRequest( + http.MethodPut, + fmt.Sprintf("/%s/_ccr/follow", followerIndex), + bytes.NewBufferString(fmt.Sprintf(followerSetting, es1Builder.Ref().Name, elasticsearch.DataIntegrityIndex)), + ) + require.NoError(t, err) + resp, err := esClient.Request(context.Background(), followerCreation) + require.NoError(t, err) + defer resp.Body.Close() + }, + }, + test.Step{ + Name: "Check data in the second cluster", + Test: test.Eventually(func() error { + return elasticsearch.NewDataIntegrityCheck(k, es2Builder).ForIndex(followerIndex).Verify() + }), + }, + es1LicenseTestContext.DeleteEnterpriseLicenseSecret(trialSecretName), + } + } + + test.Sequence(before, stepsFn, es1Builder, es2Builder).RunSequential(t) +} + +func checkRemoteClusterSeed(k *test.K8sClient, clientES, remoteES elasticsearch.Builder, expectedPort int) func() error { + return func() error { + esClient, err := elasticsearch.NewElasticsearchClient(clientES.Elasticsearch, k) + if err != nil { + return err + } + settings, err := esClient.GetRemoteClusterSettings(context.Background()) + if err != nil { + return err + } + persistentSettings := settings.PersistentSettings + if persistentSettings == nil { + return fmt.Errorf("no persistent settings found in client cluster %s/%s", clientES.Elasticsearch.Namespace, clientES.Elasticsearch.Name) + } + cluster, ok := persistentSettings.Cluster.RemoteClusters[remoteES.Name()] + if !ok { + return fmt.Errorf("client cluster %s not found in persistent settings", remoteES.Name()) + } + if len(cluster.Seeds) == 0 { + return fmt.Errorf("no seed for client cluster %s found in persistent settings", remoteES.Name()) + } + expectedSuffix := fmt.Sprintf(":%d", expectedPort) + for _, seed := range cluster.Seeds { + if !strings.HasSuffix(seed, expectedSuffix) { + return fmt.Errorf("client cluster seed must end with %s, found \"%s\"", expectedSuffix, seed) + } + } + return nil + } +} diff --git a/test/e2e/test/elasticsearch/builder.go b/test/e2e/test/elasticsearch/builder.go index 71654a8036..4718284ade 100644 --- a/test/e2e/test/elasticsearch/builder.go +++ b/test/e2e/test/elasticsearch/builder.go @@ -159,6 +159,22 @@ func (b Builder) WithRemoteCluster(remoteEs Builder) Builder { return b } +func (b Builder) WithRemoteClusterAPIKey(remoteEs Builder, apiKey *esv1.RemoteClusterAPIKey) Builder { + b.Elasticsearch.Spec.RemoteClusters = + append(b.Elasticsearch.Spec.RemoteClusters, + esv1.RemoteCluster{ + Name: remoteEs.Ref().Name, + ElasticsearchRef: remoteEs.LocalRef(), + APIKey: apiKey, + }) + return b +} + +func (b Builder) WithRemoteClusterServer() Builder { + b.Elasticsearch.Spec.RemoteClusterServer.Enabled = true + return b +} + func (b Builder) WithNamespace(namespace string) Builder { b.Elasticsearch.ObjectMeta.Namespace = namespace return b From 875c1694468e23e3c1c1f976a5ae7a04aaa7fd97 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Fri, 11 Oct 2024 12:00:08 +0200 Subject: [PATCH 07/21] [E2E] Also attempt to search + more details about errors --- pkg/controller/elasticsearch/client/model.go | 42 ++++++++++++++++++-- test/e2e/es/remote_cluster_test.go | 14 +++++-- test/e2e/test/elasticsearch/checks_data.go | 4 ++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/pkg/controller/elasticsearch/client/model.go b/pkg/controller/elasticsearch/client/model.go index 062f227795..ba0c9edef4 100644 --- a/pkg/controller/elasticsearch/client/model.go +++ b/pkg/controller/elasticsearch/client/model.go @@ -395,10 +395,44 @@ type Hits struct { // SearchResults are the results returned from a _search. type SearchResults struct { - Took int - Hits Hits `json:"hits"` - Shards json.RawMessage // model when needed - Aggs map[string]json.RawMessage // model when needed + Took int + Hits Hits `json:"hits"` + Cluster *Cluster `json:"_clusters,omitempty"` + Shards json.RawMessage // model when needed + Aggs map[string]json.RawMessage // model when needed +} + +// Cluster models the Elasticsearch response for searches that involve remote clusters. +// It can be used to provide more details about a failure. +type Cluster struct { + Details map[string]ClusterDetail `json:"details"` +} + +func (c *Cluster) Failures() string { + if c == nil { + return "" + } + failures := make([]string, 0, len(c.Details)) + for name, detail := range c.Details { + for _, failure := range detail.Failures { + failures = append(failures, fmt.Sprintf("%s: %+v", name, failure)) + } + } + if len(failures) == 0 { + return "" + } + return strings.Join(failures, ",") +} + +type ClusterDetail struct { + Status string `json:"status"` + Failures []struct { + Shard int `json:"shard"` + Reason struct { + Type string `json:"type"` + Reason string `json:"reason"` + } `json:"reason"` + } `json:"failures"` } // ShutdownType is the set of different shutdown operation types supported by Elasticsearch. diff --git a/test/e2e/es/remote_cluster_test.go b/test/e2e/es/remote_cluster_test.go index 6ecd58d3a3..7d020ca8b7 100644 --- a/test/e2e/es/remote_cluster_test.go +++ b/test/e2e/es/remote_cluster_test.go @@ -176,6 +176,7 @@ func TestRemoteClusterWithAPIKeys(t *testing.T) { } followerIndex := "data-integrity-check-follower" + remoteIndexName := fmt.Sprintf("%s:%s", es1Builder.Name(), elasticsearch.DataIntegrityIndex) stepsFn := func(k *test.K8sClient) test.StepList { return test.StepList{ // Init license test context @@ -194,8 +195,7 @@ func TestRemoteClusterWithAPIKeys(t *testing.T) { test.Step{ Name: "Add some data to the first cluster", Test: func(t *testing.T) { - // Always enable soft deletes on test index. This is required to create follower indices but disabled by default on 6.x - require.NoError(t, elasticsearch.NewDataIntegrityCheck(k, es1Builder).WithSoftDeletesEnabled(true).Init()) + require.NoError(t, elasticsearch.NewDataIntegrityCheck(k, es1Builder).Init()) }, }, test.Step{ @@ -203,7 +203,13 @@ func TestRemoteClusterWithAPIKeys(t *testing.T) { Test: test.Eventually(checkRemoteClusterSeed(k, es2Builder, es1Builder, 9443)), }, test.Step{ - Name: "Create a follower index on the second cluster", + Name: fmt.Sprintf("Check we can read remote index %s from the client cluster", remoteIndexName), + Test: test.Eventually(func() error { + return elasticsearch.NewDataIntegrityCheck(k, es2Builder).ForIndex(remoteIndexName).Verify() + }), + }, + test.Step{ + Name: "Create a follower index in the client cluster", Test: func(t *testing.T) { esClient, err := elasticsearch.NewElasticsearchClient(es2Builder.Elasticsearch, k) require.NoError(t, err) @@ -220,7 +226,7 @@ func TestRemoteClusterWithAPIKeys(t *testing.T) { }, }, test.Step{ - Name: "Check data in the second cluster", + Name: "Check data in the following index in the client cluster", Test: test.Eventually(func() error { return elasticsearch.NewDataIntegrityCheck(k, es2Builder).ForIndex(followerIndex).Verify() }), diff --git a/test/e2e/test/elasticsearch/checks_data.go b/test/e2e/test/elasticsearch/checks_data.go index 18317fdabd..c4d42c75de 100644 --- a/test/e2e/test/elasticsearch/checks_data.go +++ b/test/e2e/test/elasticsearch/checks_data.go @@ -155,6 +155,10 @@ func (dc *DataIntegrityCheck) Verify() error { if err != nil { return err } + // Check if we have remote cluster failures. + if remoteClusterFailures := results.Cluster.Failures(); len(remoteClusterFailures) > 0 { + return fmt.Errorf("remote cluster failure: %s", remoteClusterFailures) + } // the overall count should be the same if len(results.Hits.Hits) != dc.docCount { return fmt.Errorf("expected %d got %d, data loss", dc.docCount, len(results.Hits.Hits)) From c1223bdf0bcc3fe470f3918e0c3644ed3b1bdf52 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Fri, 11 Oct 2024 15:42:01 +0200 Subject: [PATCH 08/21] Only delete the keystore which has been initially loaded --- pkg/controller/remotecluster/keystore.go | 27 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/pkg/controller/remotecluster/keystore.go b/pkg/controller/remotecluster/keystore.go index 7529bdc361..04612cd885 100644 --- a/pkg/controller/remotecluster/keystore.go +++ b/pkg/controller/remotecluster/keystore.go @@ -43,6 +43,10 @@ type APIKeyStore struct { aliases map[string]AliasValue // keys maps the ID of an API Key (not its name), to the encoded cross-cluster API key. keys map[string]string + // resourceVersion is the ResourceVersion as observed when the Secret has been loaded. + resourceVersion string + // uid is the UID of the Secret as observed when the Secret has been loaded. + uid types.UID } type AliasValue struct { @@ -102,8 +106,10 @@ func LoadAPIKeyStore(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearc keys[strings[1]] = string(encodedAPIKey) } return &APIKeyStore{ - aliases: aliases, - keys: keys, + aliases: aliases, + keys: keys, + resourceVersion: keyStoreSecret.ResourceVersion, + uid: keyStoreSecret.UID, }, nil } @@ -152,19 +158,26 @@ func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elas Namespace: owner.Namespace, } if aks.IsEmpty() { - // Check if the Secret does exist. - currentSecret := corev1.Secret{} - if err := c.Get(ctx, secretName, ¤tSecret); err != nil { + // Check if the Secret still exist. + if err := c.Get(ctx, secretName, &corev1.Secret{}); err != nil { if errors.IsNotFound(err) { // Secret does not exist. return nil } return err } - // Delete the Secret we just detected above. + // Delete the Secret used to load the current state. + deleteOptions := make([]client.DeleteOption, 0, 2) + if aks.uid != "" { + deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &aks.uid}}) + } + if aks.resourceVersion != "" { + deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &aks.resourceVersion}}) + } + if err := c.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName.Name, Namespace: secretName.Namespace}}, - &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: ¤tSecret.UID}}, + deleteOptions..., ); err != nil { return err } From ab442d1e8d8e2c74276bd7c0a623b8def7cf6edb Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Fri, 11 Oct 2024 16:51:12 +0200 Subject: [PATCH 09/21] Add API keystore Secret expectations --- pkg/controller/remotecluster/apikey.go | 21 +-- pkg/controller/remotecluster/controller.go | 25 +-- .../remotecluster/controller_test.go | 3 + .../remotecluster/keystore/changes_tracker.go | 107 ++++++++++++ .../remotecluster/{ => keystore}/keystore.go | 152 +++++++++++++----- .../{ => keystore}/keystore_test.go | 23 +-- .../remotecluster/keystore/provider.go | 87 ++++++++++ pkg/controller/remotecluster/secret.go | 4 +- pkg/controller/remotecluster/watches.go | 4 +- 9 files changed, 351 insertions(+), 75 deletions(-) create mode 100644 pkg/controller/remotecluster/keystore/changes_tracker.go rename pkg/controller/remotecluster/{ => keystore}/keystore.go (53%) rename pkg/controller/remotecluster/{ => keystore}/keystore_test.go (92%) create mode 100644 pkg/controller/remotecluster/keystore/provider.go diff --git a/pkg/controller/remotecluster/apikey.go b/pkg/controller/remotecluster/apikey.go index 75b595c73e..332168646e 100644 --- a/pkg/controller/remotecluster/apikey.go +++ b/pkg/controller/remotecluster/apikey.go @@ -8,6 +8,8 @@ import ( "context" "fmt" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" @@ -28,14 +30,8 @@ func reconcileAPIKeys( clientES *esv1.Elasticsearch, // the remote Elasticsearch cluster which is going to act as the client, where the API keys are going to be store in the keystore Secret remoteClusters []esv1.RemoteCluster, // the expected API keys for that client cluster esClient esclient.Client, // ES client for the remote cluster which is going to act as the client + keystoreProvider *keystore.Provider, ) error { - // clientClusterAPIKeyStore is used to reconcile encoded API keys in the client cluster, to inject new API keys - // or to delete the ones which are no longer needed. - clientClusterAPIKeyStore, err := LoadAPIKeyStore(ctx, c, clientES) - if err != nil { - return err - } - log := ulog.FromContext(ctx).WithValues( "local_namespace", reconciledES.Namespace, "local_name", reconciledES.Name, @@ -43,6 +39,13 @@ func reconcileAPIKeys( "remote_name", clientES.Name, ) + // clientClusterAPIKeyStore is used to reconcile encoded API keys in the client cluster, to inject new API keys + // or to delete the ones which are no longer needed. + clientClusterAPIKeyStore, err := keystoreProvider.ForCluster(ctx, log, clientES) + if err != nil { + return err + } + // Maintain a list of the expected API keys for that specific client cluster, to detect the ones which are no longer expected in the reconciled cluster. expectedKeysInReconciledES := sets.New[string]() // Same for the aliases @@ -119,7 +122,7 @@ func createAPIKey( esClient esclient.Client, clientES *esv1.Elasticsearch, expectedHash string, - clientClusterAPIKeyStore *APIKeyStore, + clientClusterAPIKeyStore *keystore.APIKeyStore, reconciledES *esv1.Elasticsearch, ) error { // Active API key not found, let's create a new one. @@ -142,7 +145,7 @@ func maybeUpdateAPIKey( ctx context.Context, log logr.Logger, esClient esclient.Client, - clientClusterAPIKeyStore *APIKeyStore, + clientClusterAPIKeyStore *keystore.APIKeyStore, remoteCluster esv1.RemoteCluster, activeAPIKey *esclient.CrossClusterAPIKey, apiKeyName string, diff --git a/pkg/controller/remotecluster/controller.go b/pkg/controller/remotecluster/controller.go index 0d14a9d1b8..6956008090 100644 --- a/pkg/controller/remotecluster/controller.go +++ b/pkg/controller/remotecluster/controller.go @@ -9,15 +9,11 @@ import ( "fmt" "time" - "k8s.io/apimachinery/pkg/util/sets" - - commonesclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/esclient" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services" - "go.elastic.co/apm/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -27,6 +23,7 @@ import ( esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/association" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common" + commonesclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/esclient" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/license" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" @@ -35,6 +32,8 @@ import ( "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/certificates/remoteca" esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac" @@ -66,6 +65,7 @@ func NewReconciler(mgr manager.Manager, accessReviewer rbac.AccessReviewer, para return &ReconcileRemoteClusters{ Client: c, accessReviewer: accessReviewer, + keystoreProvider: keystore.NewProvider(c), watches: watches.NewDynamicWatches(), recorder: mgr.GetEventRecorderFor(name), licenseChecker: license.NewLicenseChecker(c, params.OperatorNamespace), @@ -85,6 +85,7 @@ type ReconcileRemoteClusters struct { watches watches.DynamicWatches licenseChecker license.Checker esClientProvider commonesclient.Provider + keystoreProvider *keystore.Provider // iteration is the number of times this controller has run its Reconcile method iteration uint64 @@ -102,6 +103,7 @@ func (r *ReconcileRemoteClusters) Reconcile(ctx context.Context, request reconci err := r.Get(ctx, request.NamespacedName, &es) if err != nil { if errors.IsNotFound(err) { + r.keystoreProvider.ForgetCluster(request.NamespacedName) return deleteAllRemoteCa(ctx, r, request.NamespacedName) } return reconcile.Result{}, err @@ -211,7 +213,7 @@ func doReconcile( if errors.IsNotFound(err) { // Remote cluster does not exist, invalidate API keys for that client cluster. apiKeyReconciledRemoteClusters.Insert(remoteEsKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, nil, esClient)) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, nil, esClient, r.keystoreProvider)) continue } return reconcile.Result{}, err @@ -232,7 +234,7 @@ func doReconcile( delete(expectedRemoteClusters, remoteEsKey) // Invalidate API keys for that client cluster. apiKeyReconciledRemoteClusters.Insert(remoteEsKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, nil, esClient)) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, nil, esClient, r.keystoreProvider)) continue } delete(associatedRemoteCAs, remoteEsKey) @@ -265,7 +267,7 @@ func doReconcile( } // Reconcile the API Keys. apiKeyReconciledRemoteClusters.Insert(remoteEsKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, remoteClusters, esClient)) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, remoteClusters, esClient, r.keystoreProvider)) } if localClusterSupportClusterAPIKeys.IsTrue() { @@ -291,11 +293,12 @@ func doReconcile( // Delete unexpected keys in the local keystore. // ********************************************* expectedAliases := expectedAliases(localEs, expectedRemoteClusters) - apiKeyStore, err := LoadAPIKeyStore(ctx, r.Client, localEs) + apiKeyStore, err := r.keystoreProvider.ForCluster(ctx, log, localEs) if err != nil { return results.WithError(err).Aggregate() } - for alias := range apiKeyStore.aliases { + + for alias := range apiKeyStore.GetAliases() { if expectedAliases.Has(alias) { // Expected alias continue @@ -356,7 +359,7 @@ func getExpectedRemoteClusters( defer span.End() expectedRemoteClusters := make(map[types.NamespacedName][]esv1.RemoteCluster) - // Add remote clusters declared in the Spec + // AddKey remote clusters declared in the Spec for _, remoteCluster := range associatedEs.Spec.RemoteClusters { if !remoteCluster.ElasticsearchRef.IsDefined() { continue diff --git a/pkg/controller/remotecluster/controller_test.go b/pkg/controller/remotecluster/controller_test.go index b453de4041..d08de74cd4 100644 --- a/pkg/controller/remotecluster/controller_test.go +++ b/pkg/controller/remotecluster/controller_test.go @@ -10,6 +10,8 @@ import ( "slices" "testing" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" + "github.com/google/go-cmp/cmp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -902,6 +904,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { esClientProvider: func(_ context.Context, _ k8s.Client, _ net.Dialer, _ esv1.Elasticsearch) (esclient.Client, error) { return fakeESClient, nil }, + keystoreProvider: keystore.NewProvider(k8sClient), } fakeCtx := context.Background() got, err := r.Reconcile(fakeCtx, tt.args.request) diff --git a/pkg/controller/remotecluster/keystore/changes_tracker.go b/pkg/controller/remotecluster/keystore/changes_tracker.go new file mode 100644 index 0000000000..5e56947155 --- /dev/null +++ b/pkg/controller/remotecluster/keystore/changes_tracker.go @@ -0,0 +1,107 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package keystore + +import ( + "sync" +) + +// pendingChanges are changes which are expected to be eventually observed in the API keystore. +type pendingChanges struct { + // changes maps for each Alias (remote cluster name as defined in the client ES) the pending change. + changes map[string]pendingChange + mu sync.RWMutex +} + +// AddKey registers a key to be eventually persisted in the underlying Secret. +func (pc *pendingChanges) AddKey(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue string) { + if pc == nil { + return + } + pc.mu.Lock() + defer pc.mu.Unlock() + pc.changes[alias] = pendingChange{ + remoteClusterName: remoteClusterName, + remoteClusterNamespace: remoteClusterNamespace, + alias: alias, + key: key{ + keyID: keyID, + encodedValue: encodedKeyValue, + }, + } +} + +// ForgetAddKey must be called once we know that the key added with AddKey is stored in the underlying Secret. +func (pc *pendingChanges) ForgetAddKey(alias string) { + if pc == nil { + return + } + pc.mu.Lock() + defer pc.mu.Unlock() + delete(pc.changes, alias) +} + +// DeleteAlias registers the deletion of the key for the provided alias. +func (pc *pendingChanges) DeleteAlias(alias string) { + if pc == nil { + return + } + pc.mu.Lock() + defer pc.mu.Unlock() + pc.changes[alias] = pendingChange{ + alias: alias, + key: key{}, // an empty key means that this alias must be deleted + } +} + +// ForgetDeleteAlias must be called once the deletion registered with DeleteAlias has been observed in the underlying Secret. +func (pc *pendingChanges) ForgetDeleteAlias(alias string) { + if pc == nil { + return + } + pc.mu.Lock() + defer pc.mu.Unlock() + delete(pc.changes, alias) +} + +// Get returns all the pending changes. +func (pc *pendingChanges) Get() []pendingChange { + if pc == nil { + return nil + } + pc.mu.RLock() + defer pc.mu.RUnlock() + pendingChanges := make([]pendingChange, 0, len(pc.changes)) + for alias, change := range pc.changes { + pendingChange := pendingChange{ + remoteClusterName: change.remoteClusterName, + remoteClusterNamespace: change.remoteClusterNamespace, + alias: alias, + key: key{ + keyID: change.key.keyID, + encodedValue: change.key.encodedValue, + }, + } + pendingChanges = append(pendingChanges, pendingChange) + } + return pendingChanges +} + +type pendingChange struct { + remoteClusterName, remoteClusterNamespace, alias string + key key +} + +// key holds an expected key as generated by Elasticsearch, with its ID and its encoded value. +type key struct { + keyID, encodedValue string +} + +func (k *key) IsEmpty() bool { + if k == nil { + return true + } + return k.encodedValue == "" && k.keyID == "" +} diff --git a/pkg/controller/remotecluster/keystore.go b/pkg/controller/remotecluster/keystore/keystore.go similarity index 53% rename from pkg/controller/remotecluster/keystore.go rename to pkg/controller/remotecluster/keystore/keystore.go index 04612cd885..0bf920a968 100644 --- a/pkg/controller/remotecluster/keystore.go +++ b/pkg/controller/remotecluster/keystore/keystore.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remotecluster +package keystore import ( "context" @@ -10,6 +10,8 @@ import ( "fmt" "regexp" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" @@ -31,7 +33,7 @@ import ( const ( aliasesAnnotationName = "elasticsearch.k8s.elastic.co/remote-cluster-api-keys" - remoteClusterAPIKeysType = "remote-cluster-api-keys" + RemoteClusterAPIKeysType = "remote-cluster-api-keys" ) var ( @@ -39,14 +41,17 @@ var ( ) type APIKeyStore struct { + log logr.Logger // aliases maps cluster aliased with the expected key ID aliases map[string]AliasValue - // keys maps the ID of an API Key (not its name), to the encoded cross-cluster API key. - keys map[string]string + // encodedKeys maps the remote cluster alias, as define in the client cluster, to the encoded cross-cluster API key. + encodedKeys map[string]string // resourceVersion is the ResourceVersion as observed when the Secret has been loaded. resourceVersion string // uid is the UID of the Secret as observed when the Secret has been loaded. uid types.UID + // pendingChanges are the pending changes, they are used to record changes until they are observed in the underlying Secret. + pendingChanges *pendingChanges } type AliasValue struct { @@ -58,6 +63,13 @@ type AliasValue struct { ID string `json:"id"` } +func (aks *APIKeyStore) GetAliases() map[string]AliasValue { + if aks == nil { + return nil + } + return aks.aliases +} + func (aks *APIKeyStore) KeyIDFor(alias string) string { if aks == nil { return "" @@ -65,7 +77,7 @@ func (aks *APIKeyStore) KeyIDFor(alias string) string { return aks.aliases[alias].ID } -func LoadAPIKeyStore(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearch) (*APIKeyStore, error) { +func loadAPIKeyStore(ctx context.Context, log logr.Logger, c k8s.Client, owner *esv1.Elasticsearch, pendingChanges *pendingChanges) (*APIKeyStore, error) { secretName := types.NamespacedName{ Name: esv1.RemoteAPIKeysSecretName(owner.Name), Namespace: owner.Namespace, @@ -79,7 +91,8 @@ func LoadAPIKeyStore(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearc "es_name", owner.Name, ) // Return an empty store - return &APIKeyStore{}, nil + emptyKeystore := &APIKeyStore{log: log, pendingChanges: pendingChanges} + return emptyKeystore.withPendingChanges(), nil } } @@ -92,7 +105,7 @@ func LoadAPIKeyStore(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearc } // Read the current encoded cross-cluster API keys. - keys := make(map[string]string) + encodedKeys := make(map[string]string) for settingName, encodedAPIKey := range keyStoreSecret.Data { strings := credentialsSecretSettingsRegEx.FindStringSubmatch(settingName) if len(strings) != 2 { @@ -103,17 +116,63 @@ func LoadAPIKeyStore(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearc ) continue } - keys[strings[1]] = string(encodedAPIKey) + encodedKeys[strings[1]] = string(encodedAPIKey) } - return &APIKeyStore{ + apiKeyStore := &APIKeyStore{ + log: log, aliases: aliases, - keys: keys, + encodedKeys: encodedKeys, resourceVersion: keyStoreSecret.ResourceVersion, uid: keyStoreSecret.UID, - }, nil + pendingChanges: pendingChanges, + } + return apiKeyStore.withPendingChanges(), nil +} + +// withPendingChanges checks if the pending changes are reflected in the Secret. If it is the case these changes are removed from the expected changes. +// If not there are "virtually" added to the current keystore. +func (aks *APIKeyStore) withPendingChanges() *APIKeyStore { + pendingChanges := aks.pendingChanges.Get() + var pendingAdds, pendingDeletions int + for _, pendingChange := range pendingChanges { + if pendingChange.key.IsEmpty() { + if aks.KeyIDFor(pendingChange.alias) == "" { + aks.log.Info(fmt.Sprintf("Change for alias %s observed, key has been deleted in API keystore", pendingChange.alias)) + aks.pendingChanges.ForgetDeleteAlias(pendingChange.alias) + continue + } + // We are still expecting this deletion + pendingDeletions++ + aks.Delete(pendingChange.alias) + continue + } + // Check if the key is available in the underlying Secret + if keyIDInSecret := aks.KeyIDFor(pendingChange.alias); keyIDInSecret == pendingChange.key.keyID { + aks.log.Info(fmt.Sprintf("Change for alias %s observed, key %s saved in API keystore", pendingChange.alias, keyIDInSecret)) + // Forget this change + aks.pendingChanges.ForgetAddKey(pendingChange.alias) + continue + } + // Change is not reflected in the Secret yet. + pendingAdds++ + aks.update(pendingChange.remoteClusterName, pendingChange.remoteClusterNamespace, pendingChange.alias, pendingChange.key.keyID, pendingChange.key.encodedValue) + } + + if pendingAdds > 0 || pendingDeletions > 0 { + aks.log.Info("Pending changes in API keystore", "add", pendingAdds, "deletion", pendingDeletions) + } + return aks } func (aks *APIKeyStore) Update(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue string) *APIKeyStore { + // Save the change in memory + aks.pendingChanges.AddKey(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue) + // Load the change in this instance of the store + aks.update(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue) + return aks +} + +func (aks *APIKeyStore) update(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue string) { if aks.aliases == nil { aks.aliases = make(map[string]AliasValue) } @@ -122,11 +181,10 @@ func (aks *APIKeyStore) Update(remoteClusterName, remoteClusterNamespace, alias, Name: remoteClusterName, ID: keyID, } - if aks.keys == nil { - aks.keys = make(map[string]string) + if aks.encodedKeys == nil { + aks.encodedKeys = make(map[string]string) } - aks.keys[alias] = encodedKeyValue - return aks + aks.encodedKeys[alias] = encodedKeyValue } func (aks *APIKeyStore) Aliases() []string { @@ -143,8 +201,16 @@ func (aks *APIKeyStore) Aliases() []string { } func (aks *APIKeyStore) Delete(alias string) *APIKeyStore { + // Save the change in memory + aks.pendingChanges.DeleteAlias(alias) + // Load the change in this instance of the store + aks.delete(alias) + return aks +} + +func (aks *APIKeyStore) delete(alias string) *APIKeyStore { delete(aks.aliases, alias) - delete(aks.keys, alias) + delete(aks.encodedKeys, alias) return aks } @@ -152,48 +218,26 @@ const ( credentialsKeyFormat = "cluster.remote.%s.credentials" ) +// Save sync the in memory content of the API keystore into the Secret. func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearch) error { secretName := types.NamespacedName{ Name: esv1.RemoteAPIKeysSecretName(owner.Name), Namespace: owner.Namespace, } if aks.IsEmpty() { - // Check if the Secret still exist. - if err := c.Get(ctx, secretName, &corev1.Secret{}); err != nil { - if errors.IsNotFound(err) { - // Secret does not exist. - return nil - } - return err - } - // Delete the Secret used to load the current state. - deleteOptions := make([]client.DeleteOption, 0, 2) - if aks.uid != "" { - deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &aks.uid}}) - } - if aks.resourceVersion != "" { - deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &aks.resourceVersion}}) - } - - if err := c.Delete(ctx, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName.Name, Namespace: secretName.Namespace}}, - deleteOptions..., - ); err != nil { - return err - } - return nil + return aks.deleteSecret(ctx, c, secretName) } aliases, err := json.Marshal(aks.aliases) if err != nil { return err } - data := make(map[string][]byte, len(aks.keys)) - for k, v := range aks.keys { + data := make(map[string][]byte, len(aks.encodedKeys)) + for k, v := range aks.encodedKeys { data[fmt.Sprintf(credentialsKeyFormat, k)] = []byte(v) } expectedLabels := labels.AddCredentialsLabel(label.NewLabels(k8s.ExtractNamespacedName(owner))) - expectedLabels[commonv1.TypeLabelName] = remoteClusterAPIKeysType + expectedLabels[commonv1.TypeLabelName] = RemoteClusterAPIKeysType expected := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName.Name, @@ -211,6 +255,27 @@ func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elas return nil } +func (aks *APIKeyStore) deleteSecret(ctx context.Context, c k8s.Client, secretName types.NamespacedName) error { + // Delete the Secret used to load the current state. + deleteOptions := make([]client.DeleteOption, 0, 2) + if aks.uid != "" { + deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &aks.uid}}) + } + if aks.resourceVersion != "" { + deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &aks.resourceVersion}}) + } + if err := c.Delete(ctx, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName.Name, Namespace: secretName.Namespace}}, + deleteOptions..., + ); err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return nil +} + func (aks *APIKeyStore) IsEmpty() bool { if aks == nil { return true @@ -218,6 +283,7 @@ func (aks *APIKeyStore) IsEmpty() bool { return len(aks.aliases) == 0 } +// ForCluster returns func (aks *APIKeyStore) ForCluster(namespace string, name string) sets.Set[string] { aliases := sets.New[string]() if aks == nil { diff --git a/pkg/controller/remotecluster/keystore_test.go b/pkg/controller/remotecluster/keystore/keystore_test.go similarity index 92% rename from pkg/controller/remotecluster/keystore_test.go rename to pkg/controller/remotecluster/keystore/keystore_test.go index 90ca9cab41..b980eb7a06 100644 --- a/pkg/controller/remotecluster/keystore_test.go +++ b/pkg/controller/remotecluster/keystore/keystore_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remotecluster +package keystore import ( "context" @@ -21,6 +21,7 @@ import ( esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" ) const ( @@ -41,8 +42,9 @@ var ( func TestLoadAPIKeyStore(t *testing.T) { type args struct { - c k8s.Client - owner *esv1.Elasticsearch + c k8s.Client + owner *esv1.Elasticsearch + pendingChanges *pendingChanges } tests := []struct { name string @@ -75,7 +77,7 @@ func TestLoadAPIKeyStore(t *testing.T) { "rc1": {ID: "SecretKeyID1", Namespace: "ns1", Name: "es1"}, "rc2": {ID: "SecretKeyID2", Namespace: "ns2", Name: "es2"}, }, - keys: map[string]string{ + encodedKeys: map[string]string{ "rc1": "SecretKeyValue1", "rc2": "SecretKeyValue2", }, @@ -93,13 +95,16 @@ func TestLoadAPIKeyStore(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() - got, err := LoadAPIKeyStore(ctx, tt.args.c, tt.args.owner) + got, err := loadAPIKeyStore(ctx, ulog.Log, tt.args.c, tt.args.owner, tt.args.pendingChanges) if (err != nil) != tt.wantErr { - t.Errorf("LoadAPIKeyStore() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("loadAPIKeyStore() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("LoadAPIKeyStore() = %v, want %v", got, tt.want) + if !reflect.DeepEqual(got.encodedKeys, tt.want.encodedKeys) { + t.Errorf("loadAPIKeyStore().keys = %v, want.keys %v", got.encodedKeys, tt.want.encodedKeys) + } + if !reflect.DeepEqual(got.aliases, tt.want.aliases) { + t.Errorf("loadAPIKeyStore().aliases = %v, want.aliases %v", got.aliases, tt.want.aliases) } }) } @@ -189,7 +194,7 @@ func TestAPIKeyStore_Save(t *testing.T) { })}, }, { - name: "Add new keys, remove another", + name: "AddKey new keys, remove another", receiver: (&APIKeyStore{}). Update("ns1", "es1", "rc1", "keyid1", "encodedValue1"). Update("ns2", "es2", "rc2", "keyid2", "encodedValue2"). diff --git a/pkg/controller/remotecluster/keystore/provider.go b/pkg/controller/remotecluster/keystore/provider.go new file mode 100644 index 0000000000..8fd9cdb177 --- /dev/null +++ b/pkg/controller/remotecluster/keystore/provider.go @@ -0,0 +1,87 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package keystore + +import ( + "context" + "sync" + + "github.com/go-logr/logr" + + "k8s.io/apimachinery/pkg/types" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" +) + +type pendingChangesPerCluster struct { + pendingChangesPerCluster map[types.NamespacedName]*pendingChanges + mu sync.RWMutex +} + +func NewProvider(c k8s.Client) *Provider { + return &Provider{ + c: c, + pendingChangesPerCluster: pendingChangesPerCluster{ + pendingChangesPerCluster: make(map[types.NamespacedName]*pendingChanges), + }, + } +} + +type Provider struct { + c k8s.Client + pendingChangesPerCluster pendingChangesPerCluster +} + +func (p *Provider) ForgetCluster(name types.NamespacedName) { + if p == nil { + return + } + p.pendingChangesPerCluster.mu.Lock() + defer p.pendingChangesPerCluster.mu.Unlock() + delete(p.pendingChangesPerCluster.pendingChangesPerCluster, name) +} + +func (p *Provider) ForCluster(ctx context.Context, log logr.Logger, owner *esv1.Elasticsearch) (*APIKeyStore, error) { + if p == nil { + return nil, nil + } + name := types.NamespacedName{ + Namespace: owner.Namespace, + Name: owner.Name, + } + pendingChanges := p.forCluster(name) + if pendingChanges != nil { + return loadAPIKeyStore(ctx, log, p.c, owner, pendingChanges) + } + return loadAPIKeyStore(ctx, log, p.c, owner, p.newForCluster(name)) +} + +func (p *Provider) forCluster(name types.NamespacedName) *pendingChanges { + if p == nil { + return nil + } + p.pendingChangesPerCluster.mu.RLock() + defer p.pendingChangesPerCluster.mu.RUnlock() + return p.pendingChangesPerCluster.pendingChangesPerCluster[name] +} + +func (p *Provider) newForCluster(name types.NamespacedName) *pendingChanges { + if p == nil { + return nil + } + p.pendingChangesPerCluster.mu.Lock() + defer p.pendingChangesPerCluster.mu.Unlock() + // Check if another goroutine did not create the pending changes + currentPendingChanges := p.pendingChangesPerCluster.pendingChangesPerCluster[name] + if currentPendingChanges != nil { + return currentPendingChanges + } + newPendingChanges := &pendingChanges{ + changes: make(map[string]pendingChange), + } + p.pendingChangesPerCluster.pendingChangesPerCluster[name] = newPendingChanges + return newPendingChanges +} diff --git a/pkg/controller/remotecluster/secret.go b/pkg/controller/remotecluster/secret.go index 605d2d1bc2..6fffe2787a 100644 --- a/pkg/controller/remotecluster/secret.go +++ b/pkg/controller/remotecluster/secret.go @@ -39,12 +39,12 @@ func createOrUpdateCertificateAuthorities( localClusterKey := k8s.ExtractNamespacedName(local) remoteClusterKey := k8s.ExtractNamespacedName(remote) - // Add watches on the CA secret of the local cluster. + // AddKey watches on the CA secret of the local cluster. if err := addCertificatesAuthorityWatches(r, localClusterKey, remoteClusterKey); err != nil { return results.WithError(err) } - // Add watches on the CA secret of the remote cluster. + // AddKey watches on the CA secret of the remote cluster. if err := addCertificatesAuthorityWatches(r, remoteClusterKey, localClusterKey); err != nil { return results.WithError(err) } diff --git a/pkg/controller/remotecluster/watches.go b/pkg/controller/remotecluster/watches.go index a7977601f0..350b68d329 100644 --- a/pkg/controller/remotecluster/watches.go +++ b/pkg/controller/remotecluster/watches.go @@ -8,6 +8,8 @@ import ( "context" "fmt" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" corev1 "k8s.io/api/core/v1" @@ -97,7 +99,7 @@ func newRequestsFromMatchedLabels() handler.TypedMapFunc[*v1.Secret, reconcile.R } if maps.ContainsKeys(labels, label.ClusterNameLabelName, commonv1.TypeLabelName) { - if labels[commonv1.TypeLabelName] != remoteClusterAPIKeysType { + if labels[commonv1.TypeLabelName] != keystore.RemoteClusterAPIKeysType { return nil } // Remote cluster API keys Secret event. From 2a8cd2d3903019e8769c91ac9be6f0818d6b11c8 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Tue, 15 Oct 2024 10:16:57 +0200 Subject: [PATCH 10/21] Update from review --- pkg/apis/elasticsearch/v1/remote_cluster.go | 4 ++-- .../elasticsearch/client/remote_cluster.go | 4 ++-- pkg/controller/remotecluster/apikey.go | 4 ++-- pkg/controller/remotecluster/controller.go | 14 +++++++------- pkg/controller/remotecluster/keystore/keystore.go | 7 +------ 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/pkg/apis/elasticsearch/v1/remote_cluster.go b/pkg/apis/elasticsearch/v1/remote_cluster.go index ae71f5592c..c3ccd6de13 100644 --- a/pkg/apis/elasticsearch/v1/remote_cluster.go +++ b/pkg/apis/elasticsearch/v1/remote_cluster.go @@ -13,8 +13,8 @@ var ( RemoteClusterAPIKeysMinVersion = version.MinFor(8, 10, 0) ) -// SupportRemoteClusterAPIKeys returns true if this cluster supports connecting to a remote cluster using API keys. -func (es *Elasticsearch) SupportRemoteClusterAPIKeys() (*optional.Bool, error) { +// SupportsRemoteClusterAPIKeys returns true if this cluster supports connecting to a remote cluster using API keys. +func (es *Elasticsearch) SupportsRemoteClusterAPIKeys() (*optional.Bool, error) { if es == nil { return nil, nil } diff --git a/pkg/controller/elasticsearch/client/remote_cluster.go b/pkg/controller/elasticsearch/client/remote_cluster.go index e866e329b0..622cd64cbb 100644 --- a/pkg/controller/elasticsearch/client/remote_cluster.go +++ b/pkg/controller/elasticsearch/client/remote_cluster.go @@ -25,9 +25,9 @@ type RemoteClusterClient interface { UpdateCrossClusterAPIKey(context.Context, string, CrossClusterAPIKeyUpdateRequest) (CrossClusterAPIKeyUpdateResponse, error) // InvalidateCrossClusterAPIKey invalidates a cluster API Key by its name. InvalidateCrossClusterAPIKey(context.Context, string) error - // GetCrossClusterAPIKeys attempts to retrieve Cross Cluster API Keys. + // GetCrossClusterAPIKeys attempts to retrieve active Cross Cluster API Keys. // The provided string is used as the "name" parameter in the HTTP query. - // Only active API Keys are included in the response. + // Relies on the active_only parameter to only include active API Keys in the response. GetCrossClusterAPIKeys(context.Context, string) (CrossClusterAPIKeyList, error) } diff --git a/pkg/controller/remotecluster/apikey.go b/pkg/controller/remotecluster/apikey.go index 332168646e..b2f385f6ca 100644 --- a/pkg/controller/remotecluster/apikey.go +++ b/pkg/controller/remotecluster/apikey.go @@ -27,9 +27,9 @@ func reconcileAPIKeys( c k8s.Client, activeAPIKeys esclient.CrossClusterAPIKeyList, // all the API Keys in the reconciled/local cluster reconciledES *esv1.Elasticsearch, // the Elasticsearch cluster being reconciled, where the API keys must be created/invalidated - clientES *esv1.Elasticsearch, // the remote Elasticsearch cluster which is going to act as the client, where the API keys are going to be store in the keystore Secret + clientES *esv1.Elasticsearch, // the remote Elasticsearch cluster which is going to act as the client, where the API keys are going to be stored in the keystore Secret remoteClusters []esv1.RemoteCluster, // the expected API keys for that client cluster - esClient esclient.Client, // ES client for the remote cluster which is going to act as the client + esClient esclient.Client, // ES client for the reconciled cluster which is going to act as the server keystoreProvider *keystore.Provider, ) error { log := ulog.FromContext(ctx).WithValues( diff --git a/pkg/controller/remotecluster/controller.go b/pkg/controller/remotecluster/controller.go index 6956008090..37b77b617e 100644 --- a/pkg/controller/remotecluster/controller.go +++ b/pkg/controller/remotecluster/controller.go @@ -172,12 +172,12 @@ func doReconcile( activeAPIKeys esclient.CrossClusterAPIKeyList esClient esclient.Client ) - localClusterSupportClusterAPIKeys, err := localEs.SupportRemoteClusterAPIKeys() + localClusterSupportsClusterAPIKeys, err := localEs.SupportsRemoteClusterAPIKeys() if err != nil { return reconcile.Result{}, err } results := &reconciler.Results{} - if localClusterSupportClusterAPIKeys.IsTrue() { + if localClusterSupportsClusterAPIKeys.IsTrue() { // Check if the ES API is available. We need it to create, update and invalidate // API keys in this cluster. if !services.NewElasticsearchURLProvider(*localEs, r.Client).HasEndpoints() { @@ -244,23 +244,23 @@ func doReconcile( } // RCS2, first check that both the reconciled and the client clusters are compatible. - clientClusterSupportClusterAPIKeys, err := remoteEs.SupportRemoteClusterAPIKeys() + clientClusterSupportsClusterAPIKeys, err := remoteEs.SupportsRemoteClusterAPIKeys() if err != nil { results.WithError(err) continue } - if !clientClusterSupportClusterAPIKeys.IsSet() { + if !clientClusterSupportsClusterAPIKeys.IsSet() { log.Info("Client cluster version is not available in status yet, skipping API keys reconciliation") continue } - if !localClusterSupportClusterAPIKeys.IsSet() { + if !localClusterSupportsClusterAPIKeys.IsSet() { log.Info("Cluster version is not available in status yet, skipping API keys reconciliation") continue } - if clientClusterSupportClusterAPIKeys.IsFalse() && localClusterSupportClusterAPIKeys.IsTrue() { + if clientClusterSupportsClusterAPIKeys.IsFalse() && localClusterSupportsClusterAPIKeys.IsTrue() { err := fmt.Errorf("client cluster %s/%s is running version %s which does not support remote cluster keys", remoteEs.Namespace, remoteEs.Name, remoteEs.Spec.Version) log.Error(err, "cannot configure remote cluster settings") continue @@ -270,7 +270,7 @@ func doReconcile( results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, remoteClusters, esClient, r.keystoreProvider)) } - if localClusterSupportClusterAPIKeys.IsTrue() { + if localClusterSupportsClusterAPIKeys.IsTrue() { // ************************************************************** // Delete orphaned API keys from clusters which have been deleted // ************************************************************** diff --git a/pkg/controller/remotecluster/keystore/keystore.go b/pkg/controller/remotecluster/keystore/keystore.go index 0bf920a968..d13e1647e4 100644 --- a/pkg/controller/remotecluster/keystore/keystore.go +++ b/pkg/controller/remotecluster/keystore/keystore.go @@ -86,10 +86,7 @@ func loadAPIKeyStore(ctx context.Context, log logr.Logger, c k8s.Client, owner * keyStoreSecret := &corev1.Secret{} if err := c.Get(ctx, secretName, keyStoreSecret); err != nil { if errors.IsNotFound(err) { - ulog.FromContext(ctx).V(1).Info("No APIKeyStore Secret found", - "namespace", owner.Namespace, - "es_name", owner.Name, - ) + ulog.FromContext(ctx).V(1).Info("No APIKeyStore Secret found") // Return an empty store emptyKeystore := &APIKeyStore{log: log, pendingChanges: pendingChanges} return emptyKeystore.withPendingChanges(), nil @@ -111,8 +108,6 @@ func loadAPIKeyStore(ctx context.Context, log logr.Logger, c k8s.Client, owner * if len(strings) != 2 { ulog.FromContext(ctx).V(1).Info( fmt.Sprintf("Unknown remote cluster credential setting: %s", settingName), - "namespace", owner.Namespace, - "es_name", owner.Name, ) continue } From b6bf462cde84ad69d211a313d2e7f7ca8f9721c9 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Tue, 15 Oct 2024 11:57:40 +0200 Subject: [PATCH 11/21] Add support for access.search.query --- config/crds/v1/all-crds.yaml | 3 +++ .../bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml | 3 +++ .../charts/eck-operator-crds/templates/all-crds.yaml | 3 +++ docs/reference/api-docs.asciidoc | 4 +++- pkg/apis/elasticsearch/v1/remote_cluster.go | 5 +++++ pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go | 4 ++++ 6 files changed, 21 insertions(+), 1 deletion(-) diff --git a/config/crds/v1/all-crds.yaml b/config/crds/v1/all-crds.yaml index 855a9d90c7..767c686d7e 100644 --- a/config/crds/v1/all-crds.yaml +++ b/config/crds/v1/all-crds.yaml @@ -4846,6 +4846,9 @@ spec: items: type: string type: array + query: + type: object + x-kubernetes-preserve-unknown-fields: true required: - names type: object diff --git a/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml b/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml index 2e54c7a0a3..b4947c0a31 100644 --- a/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml +++ b/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml @@ -9315,6 +9315,9 @@ spec: items: type: string type: array + query: + type: object + x-kubernetes-preserve-unknown-fields: true required: - names type: object diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml index a70e70c79d..903e451014 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml @@ -4888,6 +4888,9 @@ spec: items: type: string type: array + query: + type: object + x-kubernetes-preserve-unknown-fields: true required: - names type: object diff --git a/docs/reference/api-docs.asciidoc b/docs/reference/api-docs.asciidoc index 63c0aaca9b..688945c22d 100644 --- a/docs/reference/api-docs.asciidoc +++ b/docs/reference/api-docs.asciidoc @@ -514,6 +514,7 @@ Config represents untyped YAML configuration. - xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-logstash-v1alpha1-logstashspec[$$LogstashSpec$$] - xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-maps-v1alpha1-mapsspec[$$MapsSpec$$] - xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-nodeset[$$NodeSet$$] +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-search[$$Search$$] **** @@ -1541,7 +1542,7 @@ RemoteClusterAPIKey defines a remote cluster API Key. [id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess"] === RemoteClusterAccess - +RemoteClusterAccess models the API key specification as documented in https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html .Appears In: **** @@ -1622,6 +1623,7 @@ RoleSource references roles to create in the Elasticsearch cluster. | Field | Description | *`names`* __string array__ | | *`field_security`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-fieldsecurity[$$FieldSecurity$$]__ | +| *`query`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-config[$$Config$$]__ | |=== diff --git a/pkg/apis/elasticsearch/v1/remote_cluster.go b/pkg/apis/elasticsearch/v1/remote_cluster.go index c3ccd6de13..d4b8a7529e 100644 --- a/pkg/apis/elasticsearch/v1/remote_cluster.go +++ b/pkg/apis/elasticsearch/v1/remote_cluster.go @@ -5,6 +5,7 @@ package v1 import ( + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/optional" ) @@ -65,6 +66,7 @@ type RemoteClusterAPIKey struct { Access RemoteClusterAccess `json:"access,omitempty"` } +// RemoteClusterAccess models the API key specification as documented in https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html type RemoteClusterAccess struct { // +kubebuilder:validation:Optional Search *Search `json:"search,omitempty"` @@ -78,6 +80,9 @@ type Search struct { // +kubebuilder:validation:Optional FieldSecurity *FieldSecurity `json:"field_security,omitempty"` + + // +kubebuilder:pruning:PreserveUnknownFields + Query *commonv1.Config `json:"query,omitempty"` } type FieldSecurity struct { diff --git a/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go b/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go index ceecf7bd69..09f0dd1ca5 100644 --- a/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go +++ b/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go @@ -642,6 +642,10 @@ func (in *Search) DeepCopyInto(out *Search) { *out = new(FieldSecurity) (*in).DeepCopyInto(*out) } + if in.Query != nil { + in, out := &in.Query, &out.Query + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Search. From 322530726199504488eeef9eb4b1a033e0bac97e Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Tue, 15 Oct 2024 14:28:39 +0200 Subject: [PATCH 12/21] Add support for allow_restricted_indices --- config/crds/v1/all-crds.yaml | 2 ++ .../bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml | 2 ++ .../charts/eck-operator-crds/templates/all-crds.yaml | 2 ++ docs/reference/api-docs.asciidoc | 1 + pkg/apis/elasticsearch/v1/remote_cluster.go | 4 ++++ 5 files changed, 11 insertions(+) diff --git a/config/crds/v1/all-crds.yaml b/config/crds/v1/all-crds.yaml index 767c686d7e..e37aeac69b 100644 --- a/config/crds/v1/all-crds.yaml +++ b/config/crds/v1/all-crds.yaml @@ -4828,6 +4828,8 @@ spec: type: object search: properties: + allow_restricted_indices: + type: boolean field_security: properties: except: diff --git a/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml b/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml index b4947c0a31..9daa63df4d 100644 --- a/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml +++ b/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml @@ -9297,6 +9297,8 @@ spec: type: object search: properties: + allow_restricted_indices: + type: boolean field_security: properties: except: diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml index 903e451014..4ce44874b7 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml @@ -4870,6 +4870,8 @@ spec: type: object search: properties: + allow_restricted_indices: + type: boolean field_security: properties: except: diff --git a/docs/reference/api-docs.asciidoc b/docs/reference/api-docs.asciidoc index 688945c22d..7644b1fc87 100644 --- a/docs/reference/api-docs.asciidoc +++ b/docs/reference/api-docs.asciidoc @@ -1624,6 +1624,7 @@ RoleSource references roles to create in the Elasticsearch cluster. | *`names`* __string array__ | | *`field_security`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-fieldsecurity[$$FieldSecurity$$]__ | | *`query`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-config[$$Config$$]__ | +| *`allow_restricted_indices`* __boolean__ | |=== diff --git a/pkg/apis/elasticsearch/v1/remote_cluster.go b/pkg/apis/elasticsearch/v1/remote_cluster.go index d4b8a7529e..5bdbd0030b 100644 --- a/pkg/apis/elasticsearch/v1/remote_cluster.go +++ b/pkg/apis/elasticsearch/v1/remote_cluster.go @@ -81,8 +81,12 @@ type Search struct { // +kubebuilder:validation:Optional FieldSecurity *FieldSecurity `json:"field_security,omitempty"` + // +kubebuilder:validation:Optional // +kubebuilder:pruning:PreserveUnknownFields Query *commonv1.Config `json:"query,omitempty"` + + // +kubebuilder:validation:Optional + AllowRestrictedIndices bool `json:"allow_restricted_indices,omitempty"` } type FieldSecurity struct { From 657691095e762897de05878900ea5a98ed9baa42 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Tue, 15 Oct 2024 14:55:53 +0200 Subject: [PATCH 13/21] Fix unit tests --- .../remotecluster/controller_test.go | 54 ++++++++++--------- pkg/controller/remotecluster/fixtures.go | 9 ++-- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/pkg/controller/remotecluster/controller_test.go b/pkg/controller/remotecluster/controller_test.go index d08de74cd4..5ee2fa5e11 100644 --- a/pkg/controller/remotecluster/controller_test.go +++ b/pkg/controller/remotecluster/controller_test.go @@ -132,7 +132,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { // Clusters newClusterBuilder("ns1", "es1", "8.15.0").build(), newClusterBuilder("ns1", "es2", "8.15.0"). - // es2 -> es1 + // ns1/es2 -> ns1/es1 withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). build(), []client.Object{ @@ -161,7 +161,8 @@ func TestRemoteCluster_Reconcile(t *testing.T) { getCrossClusterAPIKeys: []string{"eck-*"}, crossClusterAPIKeyCreateRequests: []esclient.CrossClusterAPIKeyCreateRequest{ { - Name: "eck-ns1-es2-generated-ns1-es1-0-with-api-key", + // ns1/es1 is expected to create an API key for ns1/es2 + Name: "eck-ns1-es2-generated-alias-from-ns1-es2-to-ns1-es1-with-api-key", CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ Metadata: map[string]interface{}{ "elasticsearch.k8s.elastic.co/config-hash": "1384987056", @@ -179,7 +180,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { // Keystore for es2 must be updated. ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"eck-ns1-es2-generated-ns1-es1-0-with-api-key-1"}}`, + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns1-es2-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns1-es2-generated-alias-from-ns1-es2-to-ns1-es1-with-api-key"}}`, }, Labels: map[string]string{ "common.k8s.elastic.co/type": "remote-cluster-api-keys", @@ -190,7 +191,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { Name: "es2-es-remote-api-keys", }, Data: map[string][]byte{ - "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-eck-ns1-es2-generated-ns1-es1-0-with-api-key-1"), + "cluster.remote.generated-alias-from-ns1-es2-to-ns1-es1-with-api-key.credentials": []byte("generated-encoded-key-from-fake-es-client-for-eck-ns1-es2-generated-alias-from-ns1-es2-to-ns1-es1-with-api-key"), }, }, }, @@ -254,7 +255,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"apikey-from-es4-to-es1"}}`, + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns4-es4-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key"}}`, }, Labels: map[string]string{ "common.k8s.elastic.co/type": "remote-cluster-api-keys", @@ -266,14 +267,14 @@ func TestRemoteCluster_Reconcile(t *testing.T) { UID: uuid.NewUUID(), }, Data: map[string][]byte{ - "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns4-es4-to-ns1-es1-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), }, }, // Assume that ns1/es1 keystore already exists, unexpected keys should be removed. &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns2-es2-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-ns4-es4-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es4"},"api-key-to-non-existent-alias":{"namespace":"nsx","name":"esx","id":"apikey-to-esx"}}`, + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns1-es1-to-ns2-es2-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-alias-from-ns1-es1-to-ns3-es3-with-api-key":{"namespace":"ns3","name":"es3","id":"apikey-to-es3"},"api-key-to-non-existent-alias":{"namespace":"nsx","name":"esx","id":"apikey-to-esx"}}`, }, Labels: map[string]string{ "common.k8s.elastic.co/type": "remote-cluster-api-keys", @@ -285,8 +286,8 @@ func TestRemoteCluster_Reconcile(t *testing.T) { UID: uuid.NewUUID(), }, Data: map[string][]byte{ - "cluster.remote.generated-ns2-es2-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), - "cluster.remote.generated-ns4-es4-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns2-es2-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns3-es3-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), // The key below should be removed "cluster.remote.api-key-to-non-existent-alias.credentials": []byte("encoded-key-for-api-for-non-existent-cluster"), }, @@ -315,8 +316,8 @@ func TestRemoteCluster_Reconcile(t *testing.T) { // API key for es4 already exists but with a wrong hash we should expect an update. APIKeys: []esclient.CrossClusterAPIKey{ { - ID: "apikey-from-es4-to-es1", - Name: "eck-ns4-es4-generated-ns1-es1-0-with-api-key", + ID: "generated-id-from-fake-es-client-eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key", + Name: "eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key", Metadata: map[string]interface{}{ "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", "elasticsearch.k8s.elastic.co/managed-by": "eck", @@ -356,7 +357,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { getCrossClusterAPIKeys: []string{"eck-*"}, invalidateCrossClusterAPIKey: []string{"eck-ns4-es4-to-ns1-es1-0-old-alias", "eck-ns5-es5-generated-ns1-es1-0-with-api-key"}, updateCrossClusterAPIKey: map[string]esclient.CrossClusterAPIKeyUpdateRequest{ - "apikey-from-es4-to-es1": { + "generated-id-from-fake-es-client-eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key": { RemoteClusterAPIKey: esv1.RemoteClusterAPIKey{}, Metadata: map[string]any{ "elasticsearch.k8s.elastic.co/config-hash": "1384987056", @@ -370,7 +371,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { // We expect 2 keys to be created for ns1/es1 crossClusterAPIKeyCreateRequests: []esclient.CrossClusterAPIKeyCreateRequest{ { - Name: "eck-ns2-es2-generated-ns1-es1-0-with-api-key", + Name: "eck-ns2-es2-generated-alias-from-ns2-es2-to-ns1-es1-with-api-key", CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ Metadata: map[string]interface{}{ "elasticsearch.k8s.elastic.co/config-hash": "1384987056", @@ -382,7 +383,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { }, }, { - Name: "eck-ns3-es3-generated-ns1-es1-0-with-api-key", + Name: "eck-ns3-es3-generated-alias-from-ns3-es3-to-ns1-es1-with-api-key", CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ Metadata: map[string]interface{}{ "elasticsearch.k8s.elastic.co/config-hash": "1384987056", @@ -400,7 +401,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns2-es2-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"}}`, + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns1-es1-to-ns2-es2-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-alias-from-ns1-es1-to-ns3-es3-with-api-key":{"namespace":"ns3","name":"es3","id":"apikey-to-es3"}}`, }, Labels: map[string]string{ "common.k8s.elastic.co/type": "remote-cluster-api-keys", @@ -411,14 +412,15 @@ func TestRemoteCluster_Reconcile(t *testing.T) { Name: "es1-es-remote-api-keys", }, Data: map[string][]byte{ - "cluster.remote.generated-ns2-es2-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns2-es2-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns3-es3-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), }, }, { // Keystore for es2 must be updated. ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"},"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"eck-ns2-es2-generated-ns1-es1-0-with-api-key-1"}}`, + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"},"generated-alias-from-ns2-es2-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns2-es2-generated-alias-from-ns2-es2-to-ns1-es1-with-api-key"}}`, }, Labels: map[string]string{ "common.k8s.elastic.co/type": "remote-cluster-api-keys", @@ -429,15 +431,15 @@ func TestRemoteCluster_Reconcile(t *testing.T) { Name: "es2-es-remote-api-keys", }, Data: map[string][]byte{ - "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), - "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-eck-ns2-es2-generated-ns1-es1-0-with-api-key-1"), + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns2-es2-to-ns1-es1-with-api-key.credentials": []byte("generated-encoded-key-from-fake-es-client-for-eck-ns2-es2-generated-alias-from-ns2-es2-to-ns1-es1-with-api-key"), }, }, { // Keystore for es3 must be created. ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"eck-ns3-es3-generated-ns1-es1-0-with-api-key-2"}}`, + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns3-es3-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns3-es3-generated-alias-from-ns3-es3-to-ns1-es1-with-api-key"}}`, }, Labels: map[string]string{ "common.k8s.elastic.co/type": "remote-cluster-api-keys", @@ -448,13 +450,13 @@ func TestRemoteCluster_Reconcile(t *testing.T) { Name: "es3-es-remote-api-keys", }, Data: map[string][]byte{ - "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-eck-ns3-es3-generated-ns1-es1-0-with-api-key-2"), + "cluster.remote.generated-alias-from-ns3-es3-to-ns1-es1-with-api-key.credentials": []byte("generated-encoded-key-from-fake-es-client-for-eck-ns3-es3-generated-alias-from-ns3-es3-to-ns1-es1-with-api-key"), }, }, { ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"apikey-from-es4-to-es1"}}`, + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns4-es4-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key"}}`, }, Labels: map[string]string{ "common.k8s.elastic.co/type": "remote-cluster-api-keys", @@ -465,7 +467,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { Name: "es4-es-remote-api-keys", }, Data: map[string][]byte{ - "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns4-es4-to-ns1-es1-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), }, }, }, @@ -548,7 +550,7 @@ func TestRemoteCluster_Reconcile(t *testing.T) { &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns2-es2-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-ns4-es4-0-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es4"},"api-key-to-non-existent-alias":{"namespace":"nsx","name":"esx","id":"apikey-to-esx"}}`, + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns1-es1-to-ns2-es2-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-alias-from-ns1-es1-to-ns3-es3-with-api-key":{"namespace":"ns3","name":"es3","id":"apikey-to-es3"},"api-key-to-non-existent-alias":{"namespace":"nsx","name":"esx","id":"apikey-to-esx"}}`, }, Labels: map[string]string{ "common.k8s.elastic.co/type": "remote-cluster-api-keys", @@ -560,8 +562,8 @@ func TestRemoteCluster_Reconcile(t *testing.T) { UID: uuid.NewUUID(), }, Data: map[string][]byte{ - "cluster.remote.generated-ns2-es2-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), - "cluster.remote.generated-ns4-es4-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns2-es2-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns3-es3-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), // The key below should be removed "cluster.remote.api-key-to-non-existent-alias.credentials": []byte("encoded-key-for-api-for-non-existent-cluster"), }, diff --git a/pkg/controller/remotecluster/fixtures.go b/pkg/controller/remotecluster/fixtures.go index 37214dbb05..7d361db704 100644 --- a/pkg/controller/remotecluster/fixtures.go +++ b/pkg/controller/remotecluster/fixtures.go @@ -42,10 +42,9 @@ type fakeESClient struct { func (f *fakeESClient) CreateCrossClusterAPIKey(ctx context.Context, crossClusterAPIKeyCreateRequest esclient.CrossClusterAPIKeyCreateRequest) (esclient.CrossClusterAPIKeyCreateResponse, error) { f.crossClusterAPIKeyCreateRequests = append(f.crossClusterAPIKeyCreateRequests, crossClusterAPIKeyCreateRequest) return esclient.CrossClusterAPIKeyCreateResponse{ - ID: fmt.Sprintf("%s-%d", crossClusterAPIKeyCreateRequest.Name, len(f.crossClusterAPIKeyCreateRequests)), + ID: fmt.Sprintf("generated-id-from-fake-es-client-%s", crossClusterAPIKeyCreateRequest.Name), Name: crossClusterAPIKeyCreateRequest.Name, - APIKey: fmt.Sprintf("api-key-for-%s-%d", crossClusterAPIKeyCreateRequest.Name, len(f.crossClusterAPIKeyCreateRequests)), - Encoded: fmt.Sprintf("encoded-key-for-%s-%d", crossClusterAPIKeyCreateRequest.Name, len(f.crossClusterAPIKeyCreateRequests)), + Encoded: fmt.Sprintf("generated-encoded-key-from-fake-es-client-for-%s", crossClusterAPIKeyCreateRequest.Name), }, nil } @@ -85,7 +84,7 @@ func newClusterBuilder(namespace, name, version string) *clusterBuilder { func (cb *clusterBuilder) withRemoteCluster(namespace, name string) *clusterBuilder { cb.remoteClusters = append(cb.remoteClusters, esv1.RemoteCluster{ - Name: fmt.Sprintf("generated-%s-%s-%d", namespace, name, len(cb.remoteClusters)), + Name: fmt.Sprintf("alias-from-%s-%s-to-%s-%s", cb.namespace, cb.name, namespace, name), ElasticsearchRef: commonv1.LocalObjectSelector{ Name: name, Namespace: namespace, @@ -97,7 +96,7 @@ func (cb *clusterBuilder) withRemoteCluster(namespace, name string) *clusterBuil func (cb *clusterBuilder) withAPIKey(namespace, name string, apiKey *esv1.RemoteClusterAPIKey) *clusterBuilder { cb.remoteClusters = append(cb.remoteClusters, esv1.RemoteCluster{ - Name: fmt.Sprintf("generated-%s-%s-%d-with-api-key", namespace, name, len(cb.remoteClusters)), + Name: fmt.Sprintf("generated-alias-from-%s-%s-to-%s-%s-with-api-key", cb.namespace, cb.name, namespace, name), ElasticsearchRef: commonv1.LocalObjectSelector{ Name: name, Namespace: namespace, From e9fd9093306334bfb4e497bb1e566265cace9e4e Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Mon, 21 Oct 2024 14:51:25 +0200 Subject: [PATCH 14/21] Apply Peter's suggestions --- config/crds/v1/all-crds.yaml | 4 +- ...search.k8s.elastic.co_elasticsearches.yaml | 4 +- .../eck-operator-crds/templates/all-crds.yaml | 4 +- docs/reference/api-docs.asciidoc | 2 +- .../elasticsearch/v1/elasticsearch_types.go | 2 +- .../elasticsearch/client/remote_cluster.go | 48 +++++----- pkg/controller/remotecluster/apikey.go | 44 ++++----- pkg/controller/remotecluster/controller.go | 90 +++++++++---------- .../remotecluster/keystore/changes_tracker.go | 20 ++--- 9 files changed, 109 insertions(+), 109 deletions(-) diff --git a/config/crds/v1/all-crds.yaml b/config/crds/v1/all-crds.yaml index e37aeac69b..4d10e560d3 100644 --- a/config/crds/v1/all-crds.yaml +++ b/config/crds/v1/all-crds.yaml @@ -4810,8 +4810,8 @@ spec: connection. properties: apiKey: - description: 'APIKey can be used to enable remote cluster using - Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + description: 'APIKey can be used to enable remote cluster access + using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' properties: access: description: Access is the name of the API Key. It is automatically diff --git a/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml b/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml index 9daa63df4d..e244512a79 100644 --- a/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml +++ b/config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml @@ -9279,8 +9279,8 @@ spec: connection. properties: apiKey: - description: 'APIKey can be used to enable remote cluster using - Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + description: 'APIKey can be used to enable remote cluster access + using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' properties: access: description: Access is the name of the API Key. It is automatically diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml index 4ce44874b7..ac63663d05 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml @@ -4852,8 +4852,8 @@ spec: connection. properties: apiKey: - description: 'APIKey can be used to enable remote cluster using - Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + description: 'APIKey can be used to enable remote cluster access + using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' properties: access: description: Access is the name of the API Key. It is automatically diff --git a/docs/reference/api-docs.asciidoc b/docs/reference/api-docs.asciidoc index 7644b1fc87..f62c496573 100644 --- a/docs/reference/api-docs.asciidoc +++ b/docs/reference/api-docs.asciidoc @@ -1518,7 +1518,7 @@ RemoteCluster declares a remote Elasticsearch cluster connection. | *`name`* __string__ | Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters. | *`elasticsearchRef`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-localobjectselector[$$LocalObjectSelector$$]__ | ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. -| *`apiKey`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterapikey[$$RemoteClusterAPIKey$$]__ | APIKey can be used to enable remote cluster using Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html +| *`apiKey`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterapikey[$$RemoteClusterAPIKey$$]__ | APIKey can be used to enable remote cluster access using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html |=== diff --git a/pkg/apis/elasticsearch/v1/elasticsearch_types.go b/pkg/apis/elasticsearch/v1/elasticsearch_types.go index 2fb927e433..17db901d33 100644 --- a/pkg/apis/elasticsearch/v1/elasticsearch_types.go +++ b/pkg/apis/elasticsearch/v1/elasticsearch_types.go @@ -216,7 +216,7 @@ type RemoteCluster struct { // ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. ElasticsearchRef commonv1.LocalObjectSelector `json:"elasticsearchRef,omitempty"` - // APIKey can be used to enable remote cluster using Cross-Cluster API key API: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html + // APIKey can be used to enable remote cluster access using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html // +kubebuilder:validation:Optional APIKey *RemoteClusterAPIKey `json:"apiKey,omitempty"` diff --git a/pkg/controller/elasticsearch/client/remote_cluster.go b/pkg/controller/elasticsearch/client/remote_cluster.go index 622cd64cbb..24fe6badb5 100644 --- a/pkg/controller/elasticsearch/client/remote_cluster.go +++ b/pkg/controller/elasticsearch/client/remote_cluster.go @@ -67,30 +67,6 @@ func (cl *CrossClusterAPIKeyList) Len() int { return len(cl.APIKeys) } -// GetElasticsearchName returns the name of the client cluster for which this key has been created. -func (c *CrossClusterAPIKey) GetElasticsearchName() (types.NamespacedName, error) { - if c == nil { - return types.NamespacedName{}, nil - } - esNameInMetadata, ok := c.Metadata["elasticsearch.k8s.elastic.co/name"] - if !ok { - return types.NamespacedName{}, fmt.Errorf("missing metadata in cross cluster API key: elasticsearch.k8s.elastic.co/name") - } - esNamespaceInMetadata, ok := c.Metadata["elasticsearch.k8s.elastic.co/namespace"] - if !ok { - return types.NamespacedName{}, fmt.Errorf("missing metadata in cross cluster API key: elasticsearch.k8s.elastic.co/namespace") - } - - namespacedName := types.NamespacedName{} - if esName, ok := esNameInMetadata.(string); ok { - namespacedName.Name = esName - } - if esNamespace, ok := esNamespaceInMetadata.(string); ok { - namespacedName.Namespace = esNamespace - } - return namespacedName, nil -} - // GetActiveKeyWithName returns the first active key that matches the provided name or pattern. func (cl *CrossClusterAPIKeyList) GetActiveKeyWithName(name string) *CrossClusterAPIKey { if cl == nil || cl.Len() == 0 { @@ -142,6 +118,30 @@ type CrossClusterAPIKey struct { Metadata map[string]interface{} `json:"metadata,omitempty"` } +// GetElasticsearchName returns the name of the client cluster for which this key has been created. +func (c *CrossClusterAPIKey) GetElasticsearchName() (types.NamespacedName, error) { + if c == nil { + return types.NamespacedName{}, nil + } + esNameInMetadata, ok := c.Metadata["elasticsearch.k8s.elastic.co/name"] + if !ok { + return types.NamespacedName{}, fmt.Errorf("missing metadata in cross cluster API key: elasticsearch.k8s.elastic.co/name") + } + esNamespaceInMetadata, ok := c.Metadata["elasticsearch.k8s.elastic.co/namespace"] + if !ok { + return types.NamespacedName{}, fmt.Errorf("missing metadata in cross cluster API key: elasticsearch.k8s.elastic.co/namespace") + } + + namespacedName := types.NamespacedName{} + if esName, ok := esNameInMetadata.(string); ok { + namespacedName.Name = esName + } + if esNamespace, ok := esNamespaceInMetadata.(string); ok { + namespacedName.Namespace = esNamespace + } + return namespacedName, nil +} + // RemoteClustersSettings is used to build a request to update remote clusters. type RemoteClustersSettings struct { PersistentSettings *SettingsGroup `json:"persistent,omitempty"` diff --git a/pkg/controller/remotecluster/apikey.go b/pkg/controller/remotecluster/apikey.go index b2f385f6ca..7a6387306f 100644 --- a/pkg/controller/remotecluster/apikey.go +++ b/pkg/controller/remotecluster/apikey.go @@ -26,40 +26,40 @@ func reconcileAPIKeys( ctx context.Context, c k8s.Client, activeAPIKeys esclient.CrossClusterAPIKeyList, // all the API Keys in the reconciled/local cluster - reconciledES *esv1.Elasticsearch, // the Elasticsearch cluster being reconciled, where the API keys must be created/invalidated - clientES *esv1.Elasticsearch, // the remote Elasticsearch cluster which is going to act as the client, where the API keys are going to be stored in the keystore Secret - remoteClusters []esv1.RemoteCluster, // the expected API keys for that client cluster + remoteServerES *esv1.Elasticsearch, // the Elasticsearch cluster being reconciled, where the API keys must be created/invalidated + remoteClientES *esv1.Elasticsearch, // the remote Elasticsearch cluster which is going to act as the client, where the API keys are going to be stored in the keystore Secret + remoteClusterRefs []esv1.RemoteCluster, // the expected API keys for that client cluster esClient esclient.Client, // ES client for the reconciled cluster which is going to act as the server keystoreProvider *keystore.Provider, ) error { log := ulog.FromContext(ctx).WithValues( - "local_namespace", reconciledES.Namespace, - "local_name", reconciledES.Name, - "remote_namespace", clientES.Namespace, - "remote_name", clientES.Name, + "remote_server_namespace", remoteServerES.Namespace, + "remote_server_name", remoteServerES.Name, + "remote_client_namespace", remoteClientES.Namespace, + "remote_client_name", remoteClientES.Name, ) // clientClusterAPIKeyStore is used to reconcile encoded API keys in the client cluster, to inject new API keys // or to delete the ones which are no longer needed. - clientClusterAPIKeyStore, err := keystoreProvider.ForCluster(ctx, log, clientES) + clientClusterAPIKeyStore, err := keystoreProvider.ForCluster(ctx, log, remoteClientES) if err != nil { return err } // Maintain a list of the expected API keys for that specific client cluster, to detect the ones which are no longer expected in the reconciled cluster. - expectedKeysInReconciledES := sets.New[string]() + expectedKeysInRemoteServerES := sets.New[string]() // Same for the aliases expectedAliases := sets.New[string]() activeAPIKeysNames := activeAPIKeys.KeyNames() - for _, remoteCluster := range remoteClusters { - apiKeyName := fmt.Sprintf("eck-%s-%s-%s", clientES.Namespace, clientES.Name, remoteCluster.Name) - expectedKeysInReconciledES.Insert(apiKeyName) - expectedAliases.Insert(remoteCluster.Name) - if remoteCluster.APIKey == nil { + for _, remoteClusterRef := range remoteClusterRefs { + apiKeyName := fmt.Sprintf("eck-%s-%s-%s", remoteClientES.Namespace, remoteClientES.Name, remoteClusterRef.Name) + expectedKeysInRemoteServerES.Insert(apiKeyName) + expectedAliases.Insert(remoteClusterRef.Name) + if remoteClusterRef.APIKey == nil { if activeAPIKeysNames.Has(apiKeyName) { // We found an API key for that client cluster while it is not expected to have one. // It may happen when the user switched back from API keys to the legacy remote cluster. - log.Info("Invalidating API key as remote cluster is not configured to use it", "alias", remoteCluster.Name) + log.Info("Invalidating API key as remote cluster is not configured to use it", "alias", remoteClusterRef.Name) if err := esClient.InvalidateCrossClusterAPIKey(ctx, apiKeyName); err != nil { return err } @@ -69,27 +69,27 @@ func reconcileAPIKeys( // Attempt to get an existing API Key with that key name. activeAPIKey := activeAPIKeys.GetActiveKeyWithName(apiKeyName) - expectedHash := hash.HashObject(remoteCluster.APIKey) + expectedHash := hash.HashObject(remoteClusterRef.APIKey) if activeAPIKey == nil { - if err := createAPIKey(ctx, log, remoteCluster, apiKeyName, esClient, clientES, expectedHash, clientClusterAPIKeyStore, reconciledES); err != nil { + if err := createAPIKey(ctx, log, remoteClusterRef, apiKeyName, esClient, remoteClientES, expectedHash, clientClusterAPIKeyStore, remoteServerES); err != nil { return err } } else { // If an API key already exists ensure that the access field is the expected one using the hash - if err := maybeUpdateAPIKey(ctx, log, esClient, clientClusterAPIKeyStore, remoteCluster, activeAPIKey, apiKeyName, clientES, expectedHash); err != nil { + if err := maybeUpdateAPIKey(ctx, log, esClient, clientClusterAPIKeyStore, remoteClusterRef, activeAPIKey, apiKeyName, remoteClientES, expectedHash); err != nil { return err } } } // Get all the active API keys which have been created for that client cluster. - activeAPIKeysForClientCluster, err := activeAPIKeys.ForCluster(clientES.Namespace, clientES.Name) + activeAPIKeysForClientCluster, err := activeAPIKeys.ForCluster(remoteClientES.Namespace, remoteClientES.Name) if err != nil { return err } // Invalidate all the keys related to that local cluster which are not expected. for keyName := range activeAPIKeysForClientCluster.KeyNames() { - if !expectedKeysInReconciledES.Has(keyName) { + if !expectedKeysInRemoteServerES.Has(keyName) { // Unexpected key, let's invalidate it. log.Info("Invalidating unexpected API key", "key", keyName) if err := esClient.InvalidateCrossClusterAPIKey(ctx, keyName); err != nil { @@ -99,7 +99,7 @@ func reconcileAPIKeys( } // Delete all the keys in the keystore which are not expected. - aliases := clientClusterAPIKeyStore.ForCluster(reconciledES.Namespace, reconciledES.Name) + aliases := clientClusterAPIKeyStore.ForCluster(remoteServerES.Namespace, remoteServerES.Name) for existingAlias := range aliases { if expectedAliases.Has(existingAlias) { continue @@ -108,7 +108,7 @@ func reconcileAPIKeys( } // Save the generated keys in the keystore. - if err := clientClusterAPIKeyStore.Save(ctx, c, clientES); err != nil { + if err := clientClusterAPIKeyStore.Save(ctx, c, remoteClientES); err != nil { return err } return nil diff --git a/pkg/controller/remotecluster/controller.go b/pkg/controller/remotecluster/controller.go index 37b77b617e..1e5c0eb1f0 100644 --- a/pkg/controller/remotecluster/controller.go +++ b/pkg/controller/remotecluster/controller.go @@ -137,13 +137,13 @@ func deleteAllRemoteCa(ctx context.Context, r *ReconcileRemoteClusters, es types func doReconcile( ctx context.Context, r *ReconcileRemoteClusters, - localEs *esv1.Elasticsearch, + remoteServer *esv1.Elasticsearch, ) (reconcile.Result, error) { log := ulog.FromContext(ctx) - localClusterKey := k8s.ExtractNamespacedName(localEs) + remoteServerKey := k8s.ExtractNamespacedName(remoteServer) - expectedRemoteClusters, err := getExpectedRemoteClusters(ctx, r.Client, localEs) + expectedRemoteClients, err := getExpectedRemoteClientsFor(ctx, r.Client, remoteServer) if err != nil { return reconcile.Result{}, err } @@ -152,10 +152,10 @@ func doReconcile( if err != nil { return defaultRequeue, err } - if !enabled && len(expectedRemoteClusters) > 0 { + if !enabled && len(expectedRemoteClients) > 0 { log.V(1).Info( "Remote cluster controller is an enterprise feature. Enterprise features are disabled", - "namespace", localEs.Namespace, "es_name", localEs.Name, + "namespace", remoteServer.Namespace, "es_name", remoteServer.Name, ) return reconcile.Result{}, nil } @@ -163,7 +163,7 @@ func doReconcile( // Get all the clusters to which this reconciled cluster is connected to according to the existing remote CAs. // associatedRemoteCAs is used to delete the CA certificates and cancel any trust relationships // that may have existed in the past but should not exist anymore. - associatedRemoteCAs, err := getAssociatedRemoteCAs(ctx, r.Client, localClusterKey) + associatedRemoteCAs, err := getAssociatedRemoteCAs(ctx, r.Client, remoteServerKey) if err != nil { return reconcile.Result{}, err } @@ -172,79 +172,79 @@ func doReconcile( activeAPIKeys esclient.CrossClusterAPIKeyList esClient esclient.Client ) - localClusterSupportsClusterAPIKeys, err := localEs.SupportsRemoteClusterAPIKeys() + remoteServerSupportsClusterAPIKeys, err := remoteServer.SupportsRemoteClusterAPIKeys() if err != nil { return reconcile.Result{}, err } results := &reconciler.Results{} - if localClusterSupportsClusterAPIKeys.IsTrue() { + if remoteServerSupportsClusterAPIKeys.IsTrue() { // Check if the ES API is available. We need it to create, update and invalidate // API keys in this cluster. - if !services.NewElasticsearchURLProvider(*localEs, r.Client).HasEndpoints() { + if !services.NewElasticsearchURLProvider(*remoteServer, r.Client).HasEndpoints() { log.Info("Elasticsearch API is not available yet") return results.WithResult(defaultRequeue).Aggregate() } // Create a new client - newEsClient, err := r.esClientProvider(ctx, r.Client, r.Dialer, *localEs) + newEsClient, err := r.esClientProvider(ctx, r.Client, r.Dialer, *remoteServer) if err != nil { return reconcile.Result{}, err } // Check that the API is available esClient = newEsClient // Get all the API Keys, for that specific client, on the reconciled cluster. - getCrossClusterAPIKeys, err := esClient.GetCrossClusterAPIKeys(ctx, "eck-*") + crossClusterAPIKeys, err := esClient.GetCrossClusterAPIKeys(ctx, "eck-*") if err != nil { return reconcile.Result{}, err } - activeAPIKeys = getCrossClusterAPIKeys + activeAPIKeys = crossClusterAPIKeys } - // apiKeyReconciledRemoteClusters is used to track all the client clusters for which API keys have already been reconciled. + // apiKeyReconciledRemoteClients is used to track all the client clusters for which API keys have already been reconciled. // This is used to garbage collect API keys for clusters which have been deleted and are not in expectedRemoteClusters. - apiKeyReconciledRemoteClusters := sets.New[types.NamespacedName]() + apiKeyReconciledRemoteClients := sets.New[types.NamespacedName]() // Main loop to: // 1. Create or update expected remote CA. // 2. Create or update API keys and keystores. - for remoteEsKey, remoteClusters := range expectedRemoteClusters { + for remoteClientKey, remoteClusterRefs := range expectedRemoteClients { // Get the remote/client Elasticsearch cluster associated with this local/reconciled cluster. - remoteEs := &esv1.Elasticsearch{} - if err := r.Client.Get(ctx, remoteEsKey, remoteEs); err != nil { + remoteClient := &esv1.Elasticsearch{} + if err := r.Client.Get(ctx, remoteClientKey, remoteClient); err != nil { if errors.IsNotFound(err) { // Remote cluster does not exist, invalidate API keys for that client cluster. - apiKeyReconciledRemoteClusters.Insert(remoteEsKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, nil, esClient, r.keystoreProvider)) + apiKeyReconciledRemoteClients.Insert(remoteClientKey) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) continue } return reconcile.Result{}, err } log := log.WithValues( - "local_namespace", localEs.Namespace, - "local_name", localEs.Name, - "remote_namespace", remoteEs.Namespace, - "remote_name", remoteEs.Name, + "remote_server_namespace", remoteServer.Namespace, + "remote_server", remoteServer.Name, + "remote_client_namespace", remoteClient.Namespace, + "remote_client_name", remoteClient.Name, ) - accessAllowed, err := isRemoteClusterAssociationAllowed(ctx, r.accessReviewer, localEs, remoteEs, r.recorder) + accessAllowed, err := isRemoteClusterAssociationAllowed(ctx, r.accessReviewer, remoteServer, remoteClient, r.recorder) if err != nil { return reconcile.Result{}, err } // if the remote CA exists but isn't allowed anymore, it will be deleted next if !accessAllowed { // Remove from the expected remote cluster to clean up local keystore. - delete(expectedRemoteClusters, remoteEsKey) + delete(expectedRemoteClients, remoteClientKey) // Invalidate API keys for that client cluster. - apiKeyReconciledRemoteClusters.Insert(remoteEsKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, nil, esClient, r.keystoreProvider)) + apiKeyReconciledRemoteClients.Insert(remoteClientKey) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) continue } - delete(associatedRemoteCAs, remoteEsKey) - results.WithResults(createOrUpdateCertificateAuthorities(ctx, r, localEs, remoteEs)) + delete(associatedRemoteCAs, remoteClientKey) + results.WithResults(createOrUpdateCertificateAuthorities(ctx, r, remoteServer, remoteClient)) if results.HasError() { return results.Aggregate() } // RCS2, first check that both the reconciled and the client clusters are compatible. - clientClusterSupportsClusterAPIKeys, err := remoteEs.SupportsRemoteClusterAPIKeys() + clientClusterSupportsClusterAPIKeys, err := remoteClient.SupportsRemoteClusterAPIKeys() if err != nil { results.WithError(err) continue @@ -255,22 +255,22 @@ func doReconcile( continue } - if !localClusterSupportsClusterAPIKeys.IsSet() { + if !remoteServerSupportsClusterAPIKeys.IsSet() { log.Info("Cluster version is not available in status yet, skipping API keys reconciliation") continue } - if clientClusterSupportsClusterAPIKeys.IsFalse() && localClusterSupportsClusterAPIKeys.IsTrue() { - err := fmt.Errorf("client cluster %s/%s is running version %s which does not support remote cluster keys", remoteEs.Namespace, remoteEs.Name, remoteEs.Spec.Version) + if clientClusterSupportsClusterAPIKeys.IsFalse() && remoteServerSupportsClusterAPIKeys.IsTrue() { + err := fmt.Errorf("client cluster %s/%s is running version %s which does not support remote cluster keys", remoteClient.Namespace, remoteClient.Name, remoteClient.Spec.Version) log.Error(err, "cannot configure remote cluster settings") continue } // Reconcile the API Keys. - apiKeyReconciledRemoteClusters.Insert(remoteEsKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, localEs, remoteEs, remoteClusters, esClient, r.keystoreProvider)) + apiKeyReconciledRemoteClients.Insert(remoteClientKey) + results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, remoteClusterRefs, esClient, r.keystoreProvider)) } - if localClusterSupportsClusterAPIKeys.IsTrue() { + if remoteServerSupportsClusterAPIKeys.IsTrue() { // ************************************************************** // Delete orphaned API keys from clusters which have been deleted // ************************************************************** @@ -280,7 +280,7 @@ func doReconcile( results.WithError(err) continue } - if _, exists := apiKeyReconciledRemoteClusters[clientCluster]; exists { + if _, exists := apiKeyReconciledRemoteClients[clientCluster]; exists { // API keys for that client cluster have already been reconciled, skip. continue } @@ -292,8 +292,8 @@ func doReconcile( // ********************************************* // Delete unexpected keys in the local keystore. // ********************************************* - expectedAliases := expectedAliases(localEs, expectedRemoteClusters) - apiKeyStore, err := r.keystoreProvider.ForCluster(ctx, log, localEs) + expectedAliases := expectedAliases(remoteServer, expectedRemoteClients) + apiKeyStore, err := r.keystoreProvider.ForCluster(ctx, log, remoteServer) if err != nil { return results.WithError(err).Aggregate() } @@ -307,18 +307,18 @@ func doReconcile( log.Info(fmt.Sprintf("Removing unexpected remote API key %s", alias)) apiKeyStore.Delete(alias) } - results.WithError(apiKeyStore.Save(ctx, r.Client, localEs)) + results.WithError(apiKeyStore.Save(ctx, r.Client, remoteServer)) } // Delete existing but not expected remote CA for toDelete := range associatedRemoteCAs { log.V(1).Info("Deleting remote CA", - "local_namespace", localEs.Namespace, - "local_name", localEs.Name, + "local_namespace", remoteServer.Namespace, + "local_name", remoteServer.Name, "remote_namespace", toDelete.Namespace, "remote_name", toDelete.Name, ) - results.WithError(deleteCertificateAuthorities(ctx, r, localClusterKey, toDelete)) + results.WithError(deleteCertificateAuthorities(ctx, r, remoteServerKey, toDelete)) } return results.WithResult(association.RequeueRbacCheck(r.accessReviewer)).Aggregate() } @@ -347,10 +347,10 @@ func caCertMissingError(cluster types.NamespacedName) string { return fmt.Sprintf("Cannot find CA certificate cluster %s/%s", cluster.Namespace, cluster.Name) } -// getExpectedRemoteClusters returns all the remote cluster keys for which a remote ca and an API Key should be created. +// getExpectedRemoteClientsFor returns all the remote cluster keys for which a remote ca and an API Key should be created. // The CA certificates must be copied from the remote cluster to the local one and vice versa. // The API Key is created in the remote cluster and injected in the keystore of the local cluster. -func getExpectedRemoteClusters( +func getExpectedRemoteClientsFor( ctx context.Context, c k8s.Client, associatedEs *esv1.Elasticsearch, diff --git a/pkg/controller/remotecluster/keystore/changes_tracker.go b/pkg/controller/remotecluster/keystore/changes_tracker.go index 5e56947155..7889ecbab3 100644 --- a/pkg/controller/remotecluster/keystore/changes_tracker.go +++ b/pkg/controller/remotecluster/keystore/changes_tracker.go @@ -15,6 +15,16 @@ type pendingChanges struct { mu sync.RWMutex } +type pendingChange struct { + remoteClusterName, remoteClusterNamespace, alias string + key key +} + +// key holds an expected key as generated by Elasticsearch, with its ID and its encoded value. +type key struct { + keyID, encodedValue string +} + // AddKey registers a key to be eventually persisted in the underlying Secret. func (pc *pendingChanges) AddKey(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue string) { if pc == nil { @@ -89,16 +99,6 @@ func (pc *pendingChanges) Get() []pendingChange { return pendingChanges } -type pendingChange struct { - remoteClusterName, remoteClusterNamespace, alias string - key key -} - -// key holds an expected key as generated by Elasticsearch, with its ID and its encoded value. -type key struct { - keyID, encodedValue string -} - func (k *key) IsEmpty() bool { if k == nil { return true From 8c0a6fb22880c6f9b4dda47af3cbe403c84976dc Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Mon, 21 Oct 2024 14:59:36 +0200 Subject: [PATCH 15/21] Add ForgetChangeFor --- .../remotecluster/keystore/changes_tracker.go | 14 ++------------ pkg/controller/remotecluster/keystore/keystore.go | 4 ++-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/pkg/controller/remotecluster/keystore/changes_tracker.go b/pkg/controller/remotecluster/keystore/changes_tracker.go index 7889ecbab3..7975d64dc0 100644 --- a/pkg/controller/remotecluster/keystore/changes_tracker.go +++ b/pkg/controller/remotecluster/keystore/changes_tracker.go @@ -43,8 +43,8 @@ func (pc *pendingChanges) AddKey(remoteClusterName, remoteClusterNamespace, alia } } -// ForgetAddKey must be called once we know that the key added with AddKey is stored in the underlying Secret. -func (pc *pendingChanges) ForgetAddKey(alias string) { +// ForgetChangeFor must be called once we know that the key added with AddKey or removed with DeleteAlias is reflected in the underlying Secret. +func (pc *pendingChanges) ForgetChangeFor(alias string) { if pc == nil { return } @@ -66,16 +66,6 @@ func (pc *pendingChanges) DeleteAlias(alias string) { } } -// ForgetDeleteAlias must be called once the deletion registered with DeleteAlias has been observed in the underlying Secret. -func (pc *pendingChanges) ForgetDeleteAlias(alias string) { - if pc == nil { - return - } - pc.mu.Lock() - defer pc.mu.Unlock() - delete(pc.changes, alias) -} - // Get returns all the pending changes. func (pc *pendingChanges) Get() []pendingChange { if pc == nil { diff --git a/pkg/controller/remotecluster/keystore/keystore.go b/pkg/controller/remotecluster/keystore/keystore.go index d13e1647e4..7d70405283 100644 --- a/pkg/controller/remotecluster/keystore/keystore.go +++ b/pkg/controller/remotecluster/keystore/keystore.go @@ -133,7 +133,7 @@ func (aks *APIKeyStore) withPendingChanges() *APIKeyStore { if pendingChange.key.IsEmpty() { if aks.KeyIDFor(pendingChange.alias) == "" { aks.log.Info(fmt.Sprintf("Change for alias %s observed, key has been deleted in API keystore", pendingChange.alias)) - aks.pendingChanges.ForgetDeleteAlias(pendingChange.alias) + aks.pendingChanges.ForgetChangeFor(pendingChange.alias) continue } // We are still expecting this deletion @@ -145,7 +145,7 @@ func (aks *APIKeyStore) withPendingChanges() *APIKeyStore { if keyIDInSecret := aks.KeyIDFor(pendingChange.alias); keyIDInSecret == pendingChange.key.keyID { aks.log.Info(fmt.Sprintf("Change for alias %s observed, key %s saved in API keystore", pendingChange.alias, keyIDInSecret)) // Forget this change - aks.pendingChanges.ForgetAddKey(pendingChange.alias) + aks.pendingChanges.ForgetChangeFor(pendingChange.alias) continue } // Change is not reflected in the Secret yet. From c42c3f15fdc56a48a3b1b3fc3bcf39bfff2e8210 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Mon, 21 Oct 2024 15:31:57 +0200 Subject: [PATCH 16/21] Handle conflict --- pkg/controller/remotecluster/apikey.go | 20 +++++++++---------- pkg/controller/remotecluster/controller.go | 8 ++++---- .../remotecluster/keystore/keystore.go | 19 +++++++++++++----- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/pkg/controller/remotecluster/apikey.go b/pkg/controller/remotecluster/apikey.go index 7a6387306f..d6be1c8601 100644 --- a/pkg/controller/remotecluster/apikey.go +++ b/pkg/controller/remotecluster/apikey.go @@ -8,14 +8,14 @@ import ( "context" "fmt" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" - "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/hash" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" ) @@ -31,19 +31,19 @@ func reconcileAPIKeys( remoteClusterRefs []esv1.RemoteCluster, // the expected API keys for that client cluster esClient esclient.Client, // ES client for the reconciled cluster which is going to act as the server keystoreProvider *keystore.Provider, -) error { +) *reconciler.Results { log := ulog.FromContext(ctx).WithValues( "remote_server_namespace", remoteServerES.Namespace, "remote_server_name", remoteServerES.Name, "remote_client_namespace", remoteClientES.Namespace, "remote_client_name", remoteClientES.Name, ) - + results := &reconciler.Results{} // clientClusterAPIKeyStore is used to reconcile encoded API keys in the client cluster, to inject new API keys // or to delete the ones which are no longer needed. clientClusterAPIKeyStore, err := keystoreProvider.ForCluster(ctx, log, remoteClientES) if err != nil { - return err + return results.WithError(err) } // Maintain a list of the expected API keys for that specific client cluster, to detect the ones which are no longer expected in the reconciled cluster. @@ -61,7 +61,7 @@ func reconcileAPIKeys( // It may happen when the user switched back from API keys to the legacy remote cluster. log.Info("Invalidating API key as remote cluster is not configured to use it", "alias", remoteClusterRef.Name) if err := esClient.InvalidateCrossClusterAPIKey(ctx, apiKeyName); err != nil { - return err + return results.WithError(err) } } continue @@ -72,12 +72,12 @@ func reconcileAPIKeys( expectedHash := hash.HashObject(remoteClusterRef.APIKey) if activeAPIKey == nil { if err := createAPIKey(ctx, log, remoteClusterRef, apiKeyName, esClient, remoteClientES, expectedHash, clientClusterAPIKeyStore, remoteServerES); err != nil { - return err + return results.WithError(err) } } else { // If an API key already exists ensure that the access field is the expected one using the hash if err := maybeUpdateAPIKey(ctx, log, esClient, clientClusterAPIKeyStore, remoteClusterRef, activeAPIKey, apiKeyName, remoteClientES, expectedHash); err != nil { - return err + return results.WithError(err) } } } @@ -85,7 +85,7 @@ func reconcileAPIKeys( // Get all the active API keys which have been created for that client cluster. activeAPIKeysForClientCluster, err := activeAPIKeys.ForCluster(remoteClientES.Namespace, remoteClientES.Name) if err != nil { - return err + return results.WithError(err) } // Invalidate all the keys related to that local cluster which are not expected. for keyName := range activeAPIKeysForClientCluster.KeyNames() { @@ -93,7 +93,7 @@ func reconcileAPIKeys( // Unexpected key, let's invalidate it. log.Info("Invalidating unexpected API key", "key", keyName) if err := esClient.InvalidateCrossClusterAPIKey(ctx, keyName); err != nil { - return err + return results.WithError(err) } } } diff --git a/pkg/controller/remotecluster/controller.go b/pkg/controller/remotecluster/controller.go index 1e5c0eb1f0..fa22a92364 100644 --- a/pkg/controller/remotecluster/controller.go +++ b/pkg/controller/remotecluster/controller.go @@ -213,7 +213,7 @@ func doReconcile( if errors.IsNotFound(err) { // Remote cluster does not exist, invalidate API keys for that client cluster. apiKeyReconciledRemoteClients.Insert(remoteClientKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) + results.WithResults(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) continue } return reconcile.Result{}, err @@ -234,7 +234,7 @@ func doReconcile( delete(expectedRemoteClients, remoteClientKey) // Invalidate API keys for that client cluster. apiKeyReconciledRemoteClients.Insert(remoteClientKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) + results.WithResults(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) continue } delete(associatedRemoteCAs, remoteClientKey) @@ -267,7 +267,7 @@ func doReconcile( } // Reconcile the API Keys. apiKeyReconciledRemoteClients.Insert(remoteClientKey) - results.WithError(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, remoteClusterRefs, esClient, r.keystoreProvider)) + results.WithResults(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, remoteClusterRefs, esClient, r.keystoreProvider)) } if remoteServerSupportsClusterAPIKeys.IsTrue() { @@ -307,7 +307,7 @@ func doReconcile( log.Info(fmt.Sprintf("Removing unexpected remote API key %s", alias)) apiKeyStore.Delete(alias) } - results.WithError(apiKeyStore.Save(ctx, r.Client, remoteServer)) + results.WithResults(apiKeyStore.Save(ctx, r.Client, remoteServer)) } // Delete existing but not expected remote CA diff --git a/pkg/controller/remotecluster/keystore/keystore.go b/pkg/controller/remotecluster/keystore/keystore.go index 7d70405283..cb06b4673a 100644 --- a/pkg/controller/remotecluster/keystore/keystore.go +++ b/pkg/controller/remotecluster/keystore/keystore.go @@ -13,6 +13,7 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/reconcile" commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" @@ -214,7 +215,7 @@ const ( ) // Save sync the in memory content of the API keystore into the Secret. -func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearch) error { +func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearch) *reconciler.Results { secretName := types.NamespacedName{ Name: esv1.RemoteAPIKeysSecretName(owner.Name), Namespace: owner.Namespace, @@ -223,9 +224,10 @@ func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elas return aks.deleteSecret(ctx, c, secretName) } + results := &reconciler.Results{} aliases, err := json.Marshal(aks.aliases) if err != nil { - return err + return results.WithError(err) } data := make(map[string][]byte, len(aks.encodedKeys)) for k, v := range aks.encodedKeys { @@ -245,12 +247,15 @@ func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elas Data: data, } if _, err := reconciler.ReconcileSecret(ctx, c, expected, owner); err != nil { - return err + if errors.IsConflict(err) { + return results.WithResult(reconcile.Result{Requeue: true}) + } + return results.WithError(err) } return nil } -func (aks *APIKeyStore) deleteSecret(ctx context.Context, c k8s.Client, secretName types.NamespacedName) error { +func (aks *APIKeyStore) deleteSecret(ctx context.Context, c k8s.Client, secretName types.NamespacedName) *reconciler.Results { // Delete the Secret used to load the current state. deleteOptions := make([]client.DeleteOption, 0, 2) if aks.uid != "" { @@ -259,6 +264,7 @@ func (aks *APIKeyStore) deleteSecret(ctx context.Context, c k8s.Client, secretNa if aks.resourceVersion != "" { deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &aks.resourceVersion}}) } + results := &reconciler.Results{} if err := c.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName.Name, Namespace: secretName.Namespace}}, deleteOptions..., @@ -266,7 +272,10 @@ func (aks *APIKeyStore) deleteSecret(ctx context.Context, c k8s.Client, secretNa if errors.IsNotFound(err) { return nil } - return err + if errors.IsConflict(err) { + return results.WithResult(reconcile.Result{Requeue: true}) + } + return results.WithError(err) } return nil } From 9af83ee123286f57a05262717ec2c4156b0071a0 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Mon, 21 Oct 2024 17:08:13 +0200 Subject: [PATCH 17/21] typos --- pkg/apis/elasticsearch/v1/name.go | 2 +- pkg/apis/elasticsearch/v1/remote_cluster.go | 4 ++-- pkg/controller/elasticsearch/driver/driver.go | 9 ++++----- pkg/controller/remotecluster/keystore/keystore.go | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/apis/elasticsearch/v1/name.go b/pkg/apis/elasticsearch/v1/name.go index 58d815e62e..e50daa10f7 100644 --- a/pkg/apis/elasticsearch/v1/name.go +++ b/pkg/apis/elasticsearch/v1/name.go @@ -41,7 +41,7 @@ const ( // remoteCaNameSuffix is a suffix for the secret that contains the concatenation of all the remote CAs remoteCaNameSuffix = "remote-ca" - // remoteCredentialsNameSuffix is a suffix for the secret that contains the API keys for the remote clusters. + // remoteAPIKeysNameSuffix is a suffix for the secret that contains the API keys for the remote clusters. remoteAPIKeysNameSuffix = "remote-api-keys" controllerRevisionHashLen = 10 diff --git a/pkg/apis/elasticsearch/v1/remote_cluster.go b/pkg/apis/elasticsearch/v1/remote_cluster.go index 5bdbd0030b..133db90ec1 100644 --- a/pkg/apis/elasticsearch/v1/remote_cluster.go +++ b/pkg/apis/elasticsearch/v1/remote_cluster.go @@ -35,8 +35,8 @@ func (es *Elasticsearch) HasRemoteClusterAPIKey() bool { if es == nil { return false } - for _, remoteCLuster := range es.Spec.RemoteClusters { - if remoteCLuster.APIKey != nil { + for _, remoteCluster := range es.Spec.RemoteClusters { + if remoteCluster.APIKey != nil { return true } } diff --git a/pkg/controller/elasticsearch/driver/driver.go b/pkg/controller/elasticsearch/driver/driver.go index f98d98a5ac..751996837c 100644 --- a/pkg/controller/elasticsearch/driver/driver.go +++ b/pkg/controller/elasticsearch/driver/driver.go @@ -11,18 +11,17 @@ import ( "strings" "time" + "github.com/pkg/errors" + + corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - - commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" "k8s.io/utils/ptr" controller "sigs.k8s.io/controller-runtime/pkg/reconcile" + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/association" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common" diff --git a/pkg/controller/remotecluster/keystore/keystore.go b/pkg/controller/remotecluster/keystore/keystore.go index 2ffddc62a3..a15527abc6 100644 --- a/pkg/controller/remotecluster/keystore/keystore.go +++ b/pkg/controller/remotecluster/keystore/keystore.go @@ -212,7 +212,7 @@ const ( credentialsKeyFormat = "cluster.remote.%s.credentials" ) -// Save sync the in memory content of the API keystore into the Secret. +// Save synchronizes the in memory content of the API keystore into the Secret. func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearch) *reconciler.Results { secretName := types.NamespacedName{ Name: esv1.RemoteAPIKeysSecretName(owner.Name), From 823ee1e1293ce21c19438f1ef8d4fd819cb41ebd Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Wed, 30 Oct 2024 07:58:09 +0100 Subject: [PATCH 18/21] Apply suggestions from code review Co-authored-by: Peter Brachwitz --- pkg/apis/elasticsearch/v1/elasticsearch_types.go | 4 ++-- pkg/controller/elasticsearch/driver/driver.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/apis/elasticsearch/v1/elasticsearch_types.go b/pkg/apis/elasticsearch/v1/elasticsearch_types.go index 17db901d33..38a2834c41 100644 --- a/pkg/apis/elasticsearch/v1/elasticsearch_types.go +++ b/pkg/apis/elasticsearch/v1/elasticsearch_types.go @@ -82,8 +82,8 @@ type ElasticsearchSpec struct { // Image is the Elasticsearch Docker image to deploy. Image string `json:"image,omitempty"` - // RemoteClusterServer specifies if the remote cluster server must be enabled. - // This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. + // RemoteClusterServer specifies if the remote cluster server should be enabled. + // This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. // +kubebuilder:validation:Optional RemoteClusterServer RemoteClusterServer `json:"remoteClusterServer,omitempty"` diff --git a/pkg/controller/elasticsearch/driver/driver.go b/pkg/controller/elasticsearch/driver/driver.go index 751996837c..e9beaff9cd 100644 --- a/pkg/controller/elasticsearch/driver/driver.go +++ b/pkg/controller/elasticsearch/driver/driver.go @@ -347,8 +347,8 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { keystoreSecurityContext := securitycontext.For(d.Version, true) keystoreParams.SecurityContext = &keystoreSecurityContext - // setup a keystore with secure settings in an init container, if specified by the user. - // we are also using the keystore internally for the remote cluster API keys. + // Set up a keystore with secure settings in an init container, if specified by the user. + // We are also using the keystore internally for the remote cluster API keys. remoteClusterAPIKeys, err := apiKeyStoreSecretSource(ctx, &d.ES, d.Client) if err != nil { return results.WithError(err) From e4e053a93db54be44115686d472f1043de2fd5a8 Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Wed, 30 Oct 2024 08:03:27 +0100 Subject: [PATCH 19/21] Update comments --- pkg/controller/remotecluster/controller.go | 2 +- pkg/controller/remotecluster/secret.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controller/remotecluster/controller.go b/pkg/controller/remotecluster/controller.go index fa22a92364..067a4e1d78 100644 --- a/pkg/controller/remotecluster/controller.go +++ b/pkg/controller/remotecluster/controller.go @@ -211,7 +211,7 @@ func doReconcile( remoteClient := &esv1.Elasticsearch{} if err := r.Client.Get(ctx, remoteClientKey, remoteClient); err != nil { if errors.IsNotFound(err) { - // Remote cluster does not exist, invalidate API keys for that client cluster. + // Remote client cluster does not exist, invalidate API keys for that client cluster. apiKeyReconciledRemoteClients.Insert(remoteClientKey) results.WithResults(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) continue diff --git a/pkg/controller/remotecluster/secret.go b/pkg/controller/remotecluster/secret.go index 6fffe2787a..605d2d1bc2 100644 --- a/pkg/controller/remotecluster/secret.go +++ b/pkg/controller/remotecluster/secret.go @@ -39,12 +39,12 @@ func createOrUpdateCertificateAuthorities( localClusterKey := k8s.ExtractNamespacedName(local) remoteClusterKey := k8s.ExtractNamespacedName(remote) - // AddKey watches on the CA secret of the local cluster. + // Add watches on the CA secret of the local cluster. if err := addCertificatesAuthorityWatches(r, localClusterKey, remoteClusterKey); err != nil { return results.WithError(err) } - // AddKey watches on the CA secret of the remote cluster. + // Add watches on the CA secret of the remote cluster. if err := addCertificatesAuthorityWatches(r, remoteClusterKey, localClusterKey); err != nil { return results.WithError(err) } From a35d05687822d82d0d3f2c945886769e2b5273ed Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Wed, 30 Oct 2024 08:11:08 +0100 Subject: [PATCH 20/21] make generate --- config/crds/v1/all-crds.yaml | 4 ++-- .../elasticsearch.k8s.elastic.co_elasticsearches.yaml | 4 ++-- .../charts/eck-operator-crds/templates/all-crds.yaml | 4 ++-- docs/reference/api-docs.asciidoc | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/config/crds/v1/all-crds.yaml b/config/crds/v1/all-crds.yaml index 66b34177a6..3c96bbe187 100644 --- a/config/crds/v1/all-crds.yaml +++ b/config/crds/v1/all-crds.yaml @@ -4796,8 +4796,8 @@ spec: type: object remoteClusterServer: description: |- - RemoteClusterServer specifies if the remote cluster server must be enabled. - This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. + RemoteClusterServer specifies if the remote cluster server should be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. properties: enabled: type: boolean diff --git a/config/crds/v1/resources/elasticsearch.k8s.elastic.co_elasticsearches.yaml b/config/crds/v1/resources/elasticsearch.k8s.elastic.co_elasticsearches.yaml index ca7e12e51e..973f63d65a 100644 --- a/config/crds/v1/resources/elasticsearch.k8s.elastic.co_elasticsearches.yaml +++ b/config/crds/v1/resources/elasticsearch.k8s.elastic.co_elasticsearches.yaml @@ -9265,8 +9265,8 @@ spec: type: object remoteClusterServer: description: |- - RemoteClusterServer specifies if the remote cluster server must be enabled. - This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. + RemoteClusterServer specifies if the remote cluster server should be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. properties: enabled: type: boolean diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml index 9813bc5054..9a7c0da0c5 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml @@ -4838,8 +4838,8 @@ spec: type: object remoteClusterServer: description: |- - RemoteClusterServer specifies if the remote cluster server must be enabled. - This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. + RemoteClusterServer specifies if the remote cluster server should be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. properties: enabled: type: boolean diff --git a/docs/reference/api-docs.asciidoc b/docs/reference/api-docs.asciidoc index f62c496573..824a4cb333 100644 --- a/docs/reference/api-docs.asciidoc +++ b/docs/reference/api-docs.asciidoc @@ -1333,8 +1333,8 @@ ElasticsearchSpec holds the specification of an Elasticsearch cluster. | Field | Description | *`version`* __string__ | Version of Elasticsearch. | *`image`* __string__ | Image is the Elasticsearch Docker image to deploy. -| *`remoteClusterServer`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterserver[$$RemoteClusterServer$$]__ | RemoteClusterServer specifies if the remote cluster server must be enabled. -This must be enabled if this cluster is a remote cluster which is expected to be accessed using Cross-Cluster API key APIs. +| *`remoteClusterServer`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterserver[$$RemoteClusterServer$$]__ | RemoteClusterServer specifies if the remote cluster server should be enabled. +This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. | *`http`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-httpconfig[$$HTTPConfig$$]__ | HTTP holds HTTP layer settings for Elasticsearch. | *`transport`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-transportconfig[$$TransportConfig$$]__ | Transport holds transport layer settings for Elasticsearch. | *`nodeSets`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-nodeset[$$NodeSet$$] array__ | NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. From 610423aaa7334462c54ade7db7435ebfed5b799f Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Wed, 30 Oct 2024 08:11:53 +0100 Subject: [PATCH 21/21] Fix expected license --- test/e2e/es/remote_cluster_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/e2e/es/remote_cluster_test.go b/test/e2e/es/remote_cluster_test.go index 7d020ca8b7..240b173a61 100644 --- a/test/e2e/es/remote_cluster_test.go +++ b/test/e2e/es/remote_cluster_test.go @@ -184,12 +184,10 @@ func TestRemoteClusterWithAPIKeys(t *testing.T) { es2LicenseTestContext.Init(), // Check that the first cluster is using a Platinum license es1LicenseTestContext.CheckElasticsearchLicense( - client.ElasticsearchLicenseTypePlatinum, client.ElasticsearchLicenseTypeEnterprise, ), // Check that the second cluster is using a Platinum license es1LicenseTestContext.CheckElasticsearchLicense( - client.ElasticsearchLicenseTypePlatinum, client.ElasticsearchLicenseTypeEnterprise, ), test.Step{