diff --git a/cmd/manager/main.go b/cmd/manager/main.go index eb529f56d9..1f773df265 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -83,7 +83,7 @@ import ( "github.com/elastic/cloud-on-k8s/v2/pkg/controller/logstash" lsvalidation "github.com/elastic/cloud-on-k8s/v2/pkg/controller/logstash/validation" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/maps" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remoteca" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/stackconfigpolicy" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/webhook" "github.com/elastic/cloud-on-k8s/v2/pkg/dev" @@ -899,7 +899,7 @@ func registerControllers(mgr manager.Manager, params operator.Parameters, access name string registerFunc func(manager.Manager, rbac.AccessReviewer, operator.Parameters) error }{ - {name: "RemoteCA", registerFunc: remoteca.Add}, + {name: "RemoteCA", registerFunc: remotecluster.Add}, {name: "APM-ES", registerFunc: associationctl.AddApmES}, {name: "APM-KB", registerFunc: associationctl.AddApmKibana}, {name: "KB-ES", registerFunc: associationctl.AddKibanaES}, diff --git a/config/crds/v1/all-crds.yaml b/config/crds/v1/all-crds.yaml index 5e3a2a681f..3c96bbe187 100644 --- a/config/crds/v1/all-crds.yaml +++ b/config/crds/v1/all-crds.yaml @@ -4794,6 +4794,14 @@ spec: type: string type: object type: object + remoteClusterServer: + description: |- + RemoteClusterServer specifies if the remote cluster server should be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. + properties: + enabled: + type: boolean + type: object remoteClusters: description: RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster. @@ -4801,6 +4809,55 @@ spec: description: RemoteCluster declares a remote Elasticsearch cluster connection. properties: + apiKey: + description: 'APIKey can be used to enable remote cluster access + using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + properties: + access: + description: Access is the name of the API Key. It is automatically + generated if not set or empty. + properties: + replication: + properties: + names: + items: + type: string + type: array + required: + - names + type: object + search: + properties: + allow_restricted_indices: + type: boolean + field_security: + properties: + except: + items: + type: string + type: array + grant: + items: + type: string + type: array + required: + - except + - grant + type: object + names: + items: + type: string + type: array + query: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - names + type: object + type: object + required: + - access + type: object elasticsearchRef: description: ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. diff --git a/config/crds/v1/resources/elasticsearch.k8s.elastic.co_elasticsearches.yaml b/config/crds/v1/resources/elasticsearch.k8s.elastic.co_elasticsearches.yaml index fd0617c8c1..973f63d65a 100644 --- a/config/crds/v1/resources/elasticsearch.k8s.elastic.co_elasticsearches.yaml +++ b/config/crds/v1/resources/elasticsearch.k8s.elastic.co_elasticsearches.yaml @@ -9263,6 +9263,14 @@ spec: type: string type: object type: object + remoteClusterServer: + description: |- + RemoteClusterServer specifies if the remote cluster server should be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. + properties: + enabled: + type: boolean + type: object remoteClusters: description: RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster. @@ -9270,6 +9278,55 @@ spec: description: RemoteCluster declares a remote Elasticsearch cluster connection. properties: + apiKey: + description: 'APIKey can be used to enable remote cluster access + using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + properties: + access: + description: Access is the name of the API Key. It is automatically + generated if not set or empty. + properties: + replication: + properties: + names: + items: + type: string + type: array + required: + - names + type: object + search: + properties: + allow_restricted_indices: + type: boolean + field_security: + properties: + except: + items: + type: string + type: array + grant: + items: + type: string + type: array + required: + - except + - grant + type: object + names: + items: + type: string + type: array + query: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - names + type: object + type: object + required: + - access + type: object elasticsearchRef: description: ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. diff --git a/config/recipes/remoteclusters/elasticsearch.yaml b/config/recipes/remoteclusters/elasticsearch.yaml new file mode 100644 index 0000000000..9b168359c6 --- /dev/null +++ b/config/recipes/remoteclusters/elasticsearch.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ns1 +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: cluster1 + namespace: ns1 +spec: + version: 8.15.0 + remoteClusters: + - name: to-ns2-cluster2 + elasticsearchRef: + name: cluster2 + namespace: ns2 + apiKey: + access: + search: + names: + ## This requires the sample data available at https://kibana_url/app/home#/tutorial_directory/sampleData + - kibana_sample_data_ecommerce + nodeSets: + - name: default + config: + node.store.allow_mmap: false + count: 3 +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana + namespace: ns1 +spec: +# http: +# service: +# spec: +# # expose this cluster Service with a LoadBalancer +# type: LoadBalancer + version: 8.15.0 + count: 1 + elasticsearchRef: + name: "cluster1" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ns2 +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: cluster2 + namespace: ns2 +spec: + version: 8.15.0 + ## Required for this cluster to be accessed using remote cluster API keys. + remoteClusterServer: + enabled: true + nodeSets: + - name: default + config: + node.store.allow_mmap: false + count: 3 +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana + namespace: ns2 +spec: +# http: +# service: +# spec: +# # expose this cluster Service with a LoadBalancer +# type: LoadBalancer + version: 8.15.0 + count: 1 + elasticsearchRef: + name: "cluster2" \ No newline at end of file diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml index 9424bcdf8d..9a7c0da0c5 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml @@ -4836,6 +4836,14 @@ spec: type: string type: object type: object + remoteClusterServer: + description: |- + RemoteClusterServer specifies if the remote cluster server should be enabled. + This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. + properties: + enabled: + type: boolean + type: object remoteClusters: description: RemoteClusters enables you to establish uni-directional connections to a remote Elasticsearch cluster. @@ -4843,6 +4851,55 @@ spec: description: RemoteCluster declares a remote Elasticsearch cluster connection. properties: + apiKey: + description: 'APIKey can be used to enable remote cluster access + using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html' + properties: + access: + description: Access is the name of the API Key. It is automatically + generated if not set or empty. + properties: + replication: + properties: + names: + items: + type: string + type: array + required: + - names + type: object + search: + properties: + allow_restricted_indices: + type: boolean + field_security: + properties: + except: + items: + type: string + type: array + grant: + items: + type: string + type: array + required: + - except + - grant + type: object + names: + items: + type: string + type: array + query: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - names + type: object + type: object + required: + - access + type: object elasticsearchRef: description: ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. diff --git a/docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc b/docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc index 67b802e6ba..766876c9ba 100644 --- a/docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc +++ b/docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc @@ -115,12 +115,16 @@ spec: volumeAttributes: csi.cert-manager.io/issuer-name: ca-cluster-issuer <2> csi.cert-manager.io/issuer-kind: ClusterIssuer - csi.cert-manager.io/dns-names: "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local" + csi.cert-manager.io/dns-names: "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local" <3> ---- <1> Disables the self-signed certificates generated by ECK for the transport layer. <2> The example assumes that a `ClusterIssuer` by the name of `ca-cluster-issuer` exists and a PEM encoded version of the CA certificate is available in a ConfigMap (in the example named `trust`). The CA certificate must be in a file called `ca.crt` inside the ConfigMap in the same namespace as the Elasticsearch resource. +<3> If the remote cluster server is enabled, then the DNS names must also include both: +* The DNS name for the related Kubernetes `Service`: `-es-remote-cluster.${POD_NAMESPACE}.svc` +* The Pod DNS name: `${POD_NAME}.-es-.${POD_NAMESPACE}.svc` + The following manifest is only provided to illustrate how these certificates can be configured in principle, using the trust-manager Bundle resource and cert-manager provisioned certificates: [source,yaml,subs="attributes,callouts"] diff --git a/docs/reference/api-docs.asciidoc b/docs/reference/api-docs.asciidoc index 31b20b464e..824a4cb333 100644 --- a/docs/reference/api-docs.asciidoc +++ b/docs/reference/api-docs.asciidoc @@ -514,6 +514,7 @@ Config represents untyped YAML configuration. - xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-logstash-v1alpha1-logstashspec[$$LogstashSpec$$] - xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-maps-v1alpha1-mapsspec[$$MapsSpec$$] - xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-nodeset[$$NodeSet$$] +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-search[$$Search$$] **** @@ -1332,6 +1333,8 @@ ElasticsearchSpec holds the specification of an Elasticsearch cluster. | Field | Description | *`version`* __string__ | Version of Elasticsearch. | *`image`* __string__ | Image is the Elasticsearch Docker image to deploy. +| *`remoteClusterServer`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterserver[$$RemoteClusterServer$$]__ | RemoteClusterServer specifies if the remote cluster server should be enabled. +This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. | *`http`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-httpconfig[$$HTTPConfig$$]__ | HTTP holds HTTP layer settings for Elasticsearch. | *`transport`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-transportconfig[$$TransportConfig$$]__ | Transport holds transport layer settings for Elasticsearch. | *`nodeSets`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-nodeset[$$NodeSet$$] array__ | NodeSets allow specifying groups of Elasticsearch nodes sharing the same configuration and Pod templates. @@ -1384,6 +1387,24 @@ controller has not yet processed the changes contained in the Elasticsearch spec |=== +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-fieldsecurity"] +=== FieldSecurity + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-search[$$Search$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`grant`* __string array__ | +| *`except`* __string array__ | +|=== + + [id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-filerealmsource"] === FileRealmSource @@ -1497,6 +1518,76 @@ RemoteCluster declares a remote Elasticsearch cluster connection. | *`name`* __string__ | Name is the name of the remote cluster as it is set in the Elasticsearch settings. The name is expected to be unique for each remote clusters. | *`elasticsearchRef`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-localobjectselector[$$LocalObjectSelector$$]__ | ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. +| *`apiKey`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterapikey[$$RemoteClusterAPIKey$$]__ | APIKey can be used to enable remote cluster access using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html +|=== + + +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterapikey"] +=== RemoteClusterAPIKey + +RemoteClusterAPIKey defines a remote cluster API Key. + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remotecluster[$$RemoteCluster$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`access`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess[$$RemoteClusterAccess$$]__ | Access is the name of the API Key. It is automatically generated if not set or empty. +|=== + + +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess"] +=== RemoteClusterAccess + +RemoteClusterAccess models the API key specification as documented in https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterapikey[$$RemoteClusterAPIKey$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`search`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-search[$$Search$$]__ | +| *`replication`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-replication[$$Replication$$]__ | +|=== + + +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusterserver"] +=== RemoteClusterServer + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-elasticsearchspec[$$ElasticsearchSpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`enabled`* __boolean__ | +|=== + + +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-replication"] +=== Replication + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess[$$RemoteClusterAccess$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`names`* __string array__ | |=== @@ -1517,6 +1608,26 @@ RoleSource references roles to create in the Elasticsearch cluster. |=== +[id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-search"] +=== Search + + + +.Appears In: +**** +- xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-remoteclusteraccess[$$RemoteClusterAccess$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`names`* __string array__ | +| *`field_security`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-fieldsecurity[$$FieldSecurity$$]__ | +| *`query`* __xref:{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-config[$$Config$$]__ | +| *`allow_restricted_indices`* __boolean__ | +|=== + + [id="{anchor_prefix}-jackfan.us.kg-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-selfsignedtransportcertificates"] === SelfSignedTransportCertificates diff --git a/pkg/apis/elasticsearch/v1/elasticsearch_types.go b/pkg/apis/elasticsearch/v1/elasticsearch_types.go index f02c349cce..38a2834c41 100644 --- a/pkg/apis/elasticsearch/v1/elasticsearch_types.go +++ b/pkg/apis/elasticsearch/v1/elasticsearch_types.go @@ -82,6 +82,11 @@ type ElasticsearchSpec struct { // Image is the Elasticsearch Docker image to deploy. Image string `json:"image,omitempty"` + // RemoteClusterServer specifies if the remote cluster server should be enabled. + // This must be enabled if this cluster is a remote cluster which is expected to be accessed using API key authentication. + // +kubebuilder:validation:Optional + RemoteClusterServer RemoteClusterServer `json:"remoteClusterServer,omitempty"` + // HTTP holds HTTP layer settings for Elasticsearch. // +kubebuilder:validation:Optional HTTP commonv1.HTTPConfig `json:"http,omitempty"` @@ -139,6 +144,11 @@ type ElasticsearchSpec struct { RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` } +type RemoteClusterServer struct { + // +kubebuilder:validation:Optional + Enabled bool `json:"enabled,omitempty"` +} + // VolumeClaimDeletePolicy describes the delete policy for handling PersistentVolumeClaims that hold Elasticsearch data. // Inspired by https://github.com/kubernetes/enhancements/pull/2440 type VolumeClaimDeletePolicy string @@ -206,6 +216,10 @@ type RemoteCluster struct { // ElasticsearchRef is a reference to an Elasticsearch cluster running within the same k8s cluster. ElasticsearchRef commonv1.LocalObjectSelector `json:"elasticsearchRef,omitempty"` + // APIKey can be used to enable remote cluster access using Cross-Cluster API keys: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html + // +kubebuilder:validation:Optional + APIKey *RemoteClusterAPIKey `json:"apiKey,omitempty"` + // TODO: Allow the user to specify some options (transport.compress, transport.ping_schedule) } diff --git a/pkg/apis/elasticsearch/v1/fields.go b/pkg/apis/elasticsearch/v1/fields.go index e5952994f5..033a86c86d 100644 --- a/pkg/apis/elasticsearch/v1/fields.go +++ b/pkg/apis/elasticsearch/v1/fields.go @@ -26,6 +26,10 @@ const ( NetworkPublishHost = "network.publish_host" HTTPPublishHost = "http.publish_host" + RemoteClusterEnabled = "remote_cluster_server.enabled" + RemoteClusterPublishHost = "remote_cluster.publish_host" + RemoteClusterHost = "remote_cluster.host" + NodeName = "node.name" PathData = "path.data" @@ -54,6 +58,13 @@ const ( XPackSecurityTransportSslKey = "xpack.security.transport.ssl.key" XPackSecurityTransportSslVerificationMode = "xpack.security.transport.ssl.verification_mode" + XPackSecurityRemoteClusterServerSslKey = "xpack.security.remote_cluster_server.ssl.key" + XPackSecurityRemoteClusterServerSslCertificate = "xpack.security.remote_cluster_server.ssl.certificate" + XPackSecurityRemoteClusterServerSslCertificateAuthorities = "xpack.security.remote_cluster_server.ssl.certificate_authorities" + + XPackSecurityRemoteClusterClientSslKey = "xpack.security.remote_cluster_client.ssl.enabled" + XPackSecurityRemoteClusterClientSslCertificateAuthorities = "xpack.security.remote_cluster_client.ssl.certificate_authorities" + XPackLicenseUploadTypes = "xpack.license.upload.types" // supported >= 7.6.0 used as of 7.8.1 ) diff --git a/pkg/apis/elasticsearch/v1/name.go b/pkg/apis/elasticsearch/v1/name.go index 6d30966df6..e50daa10f7 100644 --- a/pkg/apis/elasticsearch/v1/name.go +++ b/pkg/apis/elasticsearch/v1/name.go @@ -23,6 +23,7 @@ const ( policyEsConfigSecretSuffix = "policy-config" //nolint:gosec httpServiceSuffix = "http" internalHTTPServiceSuffix = "internal-http" + remoteClusterServiceSuffix = "remote-cluster" transportServiceSuffix = "transport" elasticUserSecretSuffix = "elastic-user" internalUsersSecretSuffix = "internal-users" @@ -40,6 +41,9 @@ const ( // remoteCaNameSuffix is a suffix for the secret that contains the concatenation of all the remote CAs remoteCaNameSuffix = "remote-ca" + // remoteAPIKeysNameSuffix is a suffix for the secret that contains the API keys for the remote clusters. + remoteAPIKeysNameSuffix = "remote-api-keys" + controllerRevisionHashLen = 10 ) @@ -60,6 +64,7 @@ var ( scriptsConfigMapSuffix, statefulSetTransportCertificatesSecretSuffix, remoteCaNameSuffix, + remoteAPIKeysNameSuffix, } ) @@ -136,6 +141,10 @@ func InternalHTTPService(esName string) string { return ESNamer.Suffix(esName, internalHTTPServiceSuffix) } +func RemoteClusterService(esName string) string { + return ESNamer.Suffix(esName, remoteClusterServiceSuffix) +} + func HTTPService(esName string) string { return ESNamer.Suffix(esName, httpServiceSuffix) } @@ -173,6 +182,10 @@ func RemoteCaSecretName(esName string) string { return ESNamer.Suffix(esName, remoteCaNameSuffix) } +func RemoteAPIKeysSecretName(esName string) string { + return ESNamer.Suffix(esName, remoteAPIKeysNameSuffix) +} + func FileSettingsSecretName(esName string) string { return ESNamer.Suffix(esName, fileSettingsSecretSuffix) } diff --git a/pkg/apis/elasticsearch/v1/remote_cluster.go b/pkg/apis/elasticsearch/v1/remote_cluster.go new file mode 100644 index 0000000000..133db90ec1 --- /dev/null +++ b/pkg/apis/elasticsearch/v1/remote_cluster.go @@ -0,0 +1,100 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package v1 + +import ( + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/version" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/optional" +) + +var ( + RemoteClusterAPIKeysMinVersion = version.MinFor(8, 10, 0) +) + +// SupportsRemoteClusterAPIKeys returns true if this cluster supports connecting to a remote cluster using API keys. +func (es *Elasticsearch) SupportsRemoteClusterAPIKeys() (*optional.Bool, error) { + if es == nil { + return nil, nil + } + if es.Status.Version == "" { + // This cluster is not reconciled yet. + return nil, nil + } + esVersion, err := version.Parse(es.Status.Version) + if err != nil { + return nil, err + } + return optional.NewBool(esVersion.GTE(RemoteClusterAPIKeysMinVersion)), nil +} + +// HasRemoteClusterAPIKey returns true if this cluster is connecting to a remote cluster using API keys. +func (es *Elasticsearch) HasRemoteClusterAPIKey() bool { + if es == nil { + return false + } + for _, remoteCluster := range es.Spec.RemoteClusters { + if remoteCluster.APIKey != nil { + return true + } + } + return false +} + +// RemoteClustersCount returns the number of remote clusters using only certificates and API keys. +func (es *Elasticsearch) RemoteClustersCount() (int32, int32) { + if es == nil { + return 0, 0 + } + var withoutAPIKeys, withAPIKeys int32 + for _, remoteCLuster := range es.Spec.RemoteClusters { + if remoteCLuster.APIKey == nil { + withoutAPIKeys++ + continue + } + withAPIKeys++ + } + return withoutAPIKeys, withAPIKeys +} + +// RemoteClusterAPIKey defines a remote cluster API Key. +type RemoteClusterAPIKey struct { + // Access is the name of the API Key. It is automatically generated if not set or empty. + // +kubebuilder:validation:Required + Access RemoteClusterAccess `json:"access,omitempty"` +} + +// RemoteClusterAccess models the API key specification as documented in https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-cross-cluster-api-key.html +type RemoteClusterAccess struct { + // +kubebuilder:validation:Optional + Search *Search `json:"search,omitempty"` + // +kubebuilder:validation:Optional + Replication *Replication `json:"replication,omitempty"` +} + +type Search struct { + // +kubebuilder:validation:Required + Names []string `json:"names,omitempty"` + + // +kubebuilder:validation:Optional + FieldSecurity *FieldSecurity `json:"field_security,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:pruning:PreserveUnknownFields + Query *commonv1.Config `json:"query,omitempty"` + + // +kubebuilder:validation:Optional + AllowRestrictedIndices bool `json:"allow_restricted_indices,omitempty"` +} + +type FieldSecurity struct { + Grant []string `json:"grant"` + Except []string `json:"except"` +} + +type Replication struct { + // +kubebuilder:validation:Required + Names []string `json:"names,omitempty"` +} diff --git a/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go b/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go index edb7c903e8..09f0dd1ca5 100644 --- a/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go +++ b/pkg/apis/elasticsearch/v1/zz_generated.deepcopy.go @@ -255,6 +255,7 @@ func (in *ElasticsearchSettings) DeepCopy() *ElasticsearchSettings { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ElasticsearchSpec) DeepCopyInto(out *ElasticsearchSpec) { *out = *in + out.RemoteClusterServer = in.RemoteClusterServer in.HTTP.DeepCopyInto(&out.HTTP) in.Transport.DeepCopyInto(&out.Transport) if in.NodeSets != nil { @@ -281,7 +282,9 @@ func (in *ElasticsearchSpec) DeepCopyInto(out *ElasticsearchSpec) { if in.RemoteClusters != nil { in, out := &in.RemoteClusters, &out.RemoteClusters *out = make([]RemoteCluster, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } in.Monitoring.DeepCopyInto(&out.Monitoring) if in.RevisionHistoryLimit != nil { @@ -352,6 +355,31 @@ func (in *EsMonitoringAssociation) DeepCopy() *EsMonitoringAssociation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FieldSecurity) DeepCopyInto(out *FieldSecurity) { + *out = *in + if in.Grant != nil { + in, out := &in.Grant, &out.Grant + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Except != nil { + in, out := &in.Except, &out.Except + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FieldSecurity. +func (in *FieldSecurity) DeepCopy() *FieldSecurity { + if in == nil { + return nil + } + out := new(FieldSecurity) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FileRealmSource) DeepCopyInto(out *FileRealmSource) { *out = *in @@ -492,6 +520,11 @@ func (in *NodeSet) DeepCopy() *NodeSet { func (in *RemoteCluster) DeepCopyInto(out *RemoteCluster) { *out = *in out.ElasticsearchRef = in.ElasticsearchRef + if in.APIKey != nil { + in, out := &in.APIKey, &out.APIKey + *out = new(RemoteClusterAPIKey) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteCluster. @@ -504,6 +537,82 @@ func (in *RemoteCluster) DeepCopy() *RemoteCluster { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterAPIKey) DeepCopyInto(out *RemoteClusterAPIKey) { + *out = *in + in.Access.DeepCopyInto(&out.Access) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterAPIKey. +func (in *RemoteClusterAPIKey) DeepCopy() *RemoteClusterAPIKey { + if in == nil { + return nil + } + out := new(RemoteClusterAPIKey) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterAccess) DeepCopyInto(out *RemoteClusterAccess) { + *out = *in + if in.Search != nil { + in, out := &in.Search, &out.Search + *out = new(Search) + (*in).DeepCopyInto(*out) + } + if in.Replication != nil { + in, out := &in.Replication, &out.Replication + *out = new(Replication) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterAccess. +func (in *RemoteClusterAccess) DeepCopy() *RemoteClusterAccess { + if in == nil { + return nil + } + out := new(RemoteClusterAccess) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteClusterServer) DeepCopyInto(out *RemoteClusterServer) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteClusterServer. +func (in *RemoteClusterServer) DeepCopy() *RemoteClusterServer { + if in == nil { + return nil + } + out := new(RemoteClusterServer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Replication) DeepCopyInto(out *Replication) { + *out = *in + if in.Names != nil { + in, out := &in.Names, &out.Names + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Replication. +func (in *Replication) DeepCopy() *Replication { + if in == nil { + return nil + } + out := new(Replication) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RoleSource) DeepCopyInto(out *RoleSource) { *out = *in @@ -520,6 +629,35 @@ func (in *RoleSource) DeepCopy() *RoleSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Search) DeepCopyInto(out *Search) { + *out = *in + if in.Names != nil { + in, out := &in.Names, &out.Names + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.FieldSecurity != nil { + in, out := &in.FieldSecurity, &out.FieldSecurity + *out = new(FieldSecurity) + (*in).DeepCopyInto(*out) + } + if in.Query != nil { + in, out := &in.Query, &out.Query + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Search. +func (in *Search) DeepCopy() *Search { + if in == nil { + return nil + } + out := new(Search) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SelfSignedTransportCertificates) DeepCopyInto(out *SelfSignedTransportCertificates) { *out = *in diff --git a/pkg/controller/common/keystore/resources.go b/pkg/controller/common/keystore/resources.go index 0a92f48387..0e07424b69 100644 --- a/pkg/controller/common/keystore/resources.go +++ b/pkg/controller/common/keystore/resources.go @@ -58,9 +58,10 @@ func ReconcileResources( namer name.Namer, labels map[string]string, initContainerParams InitContainerParameters, + additionalSources ...commonv1.NamespacedSecretSource, ) (*Resources, error) { // setup a volume from the user-provided secure settings secret - secretVolume, hash, err := secureSettingsVolume(ctx, r, hasKeystore, labels, namer) + secretVolume, hash, err := secureSettingsVolume(ctx, r, hasKeystore, labels, namer, additionalSources...) if err != nil { return nil, err } diff --git a/pkg/controller/common/keystore/user_secret.go b/pkg/controller/common/keystore/user_secret.go index 8b765d4259..dac92e95a6 100644 --- a/pkg/controller/common/keystore/user_secret.go +++ b/pkg/controller/common/keystore/user_secret.go @@ -44,12 +44,15 @@ func secureSettingsVolume( hasKeystore HasKeystore, labels map[string]string, namer name.Namer, + additionalSources ...commonv1.NamespacedSecretSource, ) (*volume.SecretVolume, string, error) { // setup (or remove) watches for the user-provided secret to reconcile on any change watcher := k8s.ExtractNamespacedName(hasKeystore) // user-provided Secrets referenced in the resource secretSources := WatchedSecretNames(hasKeystore) + // Additional sources, introduced to load remote cluster keys. + secretSources = append(secretSources, additionalSources...) // user-provided Secrets referenced in a StackConfigPolicy that configures the resource policySecretSources, err := stackconfigpolicy.GetSecureSettingsSecretSourcesForResources(ctx, r.K8sClient(), hasKeystore, hasKeystore.GetObjectKind().GroupVersionKind().Kind) if err != nil { diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index cc6cddf776..8a71c5b381 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -104,6 +104,18 @@ func buildGeneralNames( {IPAddress: netutil.IPToRFCForm(netutil.LoopbackFor(netutil.ToIPFamily(podIP.String())))}, } + if cluster.Spec.RemoteClusterServer.Enabled { + // Remote cluster server is enabled. Ensure that the remote cluster service name is included in the transport certificates + // since these are the ones also used in the context of remote clusters access using API keys. + generalNames = append( + generalNames, + // Remote cluster headless service + certificates.GeneralName{DNSName: fmt.Sprintf("%s.%s.svc", esv1.RemoteClusterService(cluster.Name), cluster.Namespace)}, + // Individual remote_cluster.publish_host is set to ...svc + certificates.GeneralName{DNSName: fmt.Sprintf("%s.%s.%s.svc", pod.Name, svcName, cluster.Namespace)}, + ) + } + for _, san := range cluster.Spec.Transport.TLS.SubjectAlternativeNames { if san.DNS != "" { generalNames = append(generalNames, certificates.GeneralName{DNSName: san.DNS}) diff --git a/pkg/controller/elasticsearch/client/base.go b/pkg/controller/elasticsearch/client/base.go index eff91370d7..90d08fefbd 100644 --- a/pkg/controller/elasticsearch/client/base.go +++ b/pkg/controller/elasticsearch/client/base.go @@ -94,7 +94,7 @@ func (c *baseClient) get(ctx context.Context, pathWithQuery string, out interfac return c.request(ctx, http.MethodGet, pathWithQuery, nil, out, nil) } -func (c *baseClient) put(ctx context.Context, pathWithQuery string, in, out interface{}) error { //nolint:unparam +func (c *baseClient) put(ctx context.Context, pathWithQuery string, in, out interface{}) error { return c.request(ctx, http.MethodPut, pathWithQuery, in, out, nil) } @@ -106,6 +106,10 @@ func (c *baseClient) delete(ctx context.Context, pathWithQuery string) error { return c.request(ctx, http.MethodDelete, pathWithQuery, nil, nil, nil) } +func (c *baseClient) deleteWithObjects(ctx context.Context, pathWithQuery string, in, out interface{}) error { + return c.request(ctx, http.MethodDelete, pathWithQuery, in, out, nil) +} + // request performs a new http request // // if requestObj is not nil, it's marshalled as JSON and used as the request body diff --git a/pkg/controller/elasticsearch/client/client.go b/pkg/controller/elasticsearch/client/client.go index 8729ff0383..d9e3ea61e8 100644 --- a/pkg/controller/elasticsearch/client/client.go +++ b/pkg/controller/elasticsearch/client/client.go @@ -63,6 +63,7 @@ type Client interface { DesiredNodesClient ShardLister LicenseClient + RemoteClusterClient SecurityClient // Close idle connections in the underlying http client. Close() @@ -101,10 +102,6 @@ type Client interface { GetNodesStats(ctx context.Context) (NodesStats, error) // ClusterBootstrappedForZen2 returns true if the cluster is relying on zen2 orchestration. ClusterBootstrappedForZen2(ctx context.Context) (bool, error) - // UpdateRemoteClusterSettings updates the remote clusters of a cluster. - UpdateRemoteClusterSettings(ctx context.Context, settings RemoteClustersSettings) error - // GetRemoteClusterSettings retrieves the remote clusters of a cluster. - GetRemoteClusterSettings(ctx context.Context) (RemoteClustersSettings, error) // AddVotingConfigExclusions sets the transient and persistent setting of the same name in cluster settings. // Introduced in: Elasticsearch 7.0.0 AddVotingConfigExclusions(ctx context.Context, nodeNames []string) error diff --git a/pkg/controller/elasticsearch/client/model.go b/pkg/controller/elasticsearch/client/model.go index 339075ed36..ba0c9edef4 100644 --- a/pkg/controller/elasticsearch/client/model.go +++ b/pkg/controller/elasticsearch/client/model.go @@ -378,26 +378,6 @@ type StartBasicResponse struct { ErrorMessage string `json:"error_message"` } -// RemoteClustersSettings is used to build a request to update remote clusters. -type RemoteClustersSettings struct { - PersistentSettings *SettingsGroup `json:"persistent,omitempty"` -} - -// SettingsGroup is a group of persistent settings. -type SettingsGroup struct { - Cluster RemoteClusters `json:"cluster,omitempty"` -} - -// RemoteClusters models the configuration of the remote clusters. -type RemoteClusters struct { - RemoteClusters map[string]RemoteCluster `json:"remote,omitempty"` -} - -// RemoteCluster is the set of seeds to use in a remote cluster setting. -type RemoteCluster struct { - Seeds []string `json:"seeds"` -} - // Hit represents a single search hit. type Hit struct { Index string `json:"_index"` @@ -415,10 +395,44 @@ type Hits struct { // SearchResults are the results returned from a _search. type SearchResults struct { - Took int - Hits Hits `json:"hits"` - Shards json.RawMessage // model when needed - Aggs map[string]json.RawMessage // model when needed + Took int + Hits Hits `json:"hits"` + Cluster *Cluster `json:"_clusters,omitempty"` + Shards json.RawMessage // model when needed + Aggs map[string]json.RawMessage // model when needed +} + +// Cluster models the Elasticsearch response for searches that involve remote clusters. +// It can be used to provide more details about a failure. +type Cluster struct { + Details map[string]ClusterDetail `json:"details"` +} + +func (c *Cluster) Failures() string { + if c == nil { + return "" + } + failures := make([]string, 0, len(c.Details)) + for name, detail := range c.Details { + for _, failure := range detail.Failures { + failures = append(failures, fmt.Sprintf("%s: %+v", name, failure)) + } + } + if len(failures) == 0 { + return "" + } + return strings.Join(failures, ",") +} + +type ClusterDetail struct { + Status string `json:"status"` + Failures []struct { + Shard int `json:"shard"` + Reason struct { + Type string `json:"type"` + Reason string `json:"reason"` + } `json:"reason"` + } `json:"failures"` } // ShutdownType is the set of different shutdown operation types supported by Elasticsearch. diff --git a/pkg/controller/elasticsearch/client/remote_cluster.go b/pkg/controller/elasticsearch/client/remote_cluster.go new file mode 100644 index 0000000000..24fe6badb5 --- /dev/null +++ b/pkg/controller/elasticsearch/client/remote_cluster.go @@ -0,0 +1,163 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package client + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" +) + +type RemoteClusterClient interface { + // UpdateRemoteClusterSettings updates the remote clusters of a cluster. + UpdateRemoteClusterSettings(context.Context, RemoteClustersSettings) error + // GetRemoteClusterSettings retrieves the remote clusters of a cluster. + GetRemoteClusterSettings(context.Context) (RemoteClustersSettings, error) + // CreateCrossClusterAPIKey creates a new cross cluster API Key using the provided cross cluster API key request. + CreateCrossClusterAPIKey(context.Context, CrossClusterAPIKeyCreateRequest) (CrossClusterAPIKeyCreateResponse, error) + // UpdateCrossClusterAPIKey updates the cluster API Key which matches the provided ID using the provided update request. + UpdateCrossClusterAPIKey(context.Context, string, CrossClusterAPIKeyUpdateRequest) (CrossClusterAPIKeyUpdateResponse, error) + // InvalidateCrossClusterAPIKey invalidates a cluster API Key by its name. + InvalidateCrossClusterAPIKey(context.Context, string) error + // GetCrossClusterAPIKeys attempts to retrieve active Cross Cluster API Keys. + // The provided string is used as the "name" parameter in the HTTP query. + // Relies on the active_only parameter to only include active API Keys in the response. + GetCrossClusterAPIKeys(context.Context, string) (CrossClusterAPIKeyList, error) +} + +type CrossClusterAPIKeyInvalidateRequest struct { + Name string `json:"name,omitempty"` +} + +type CrossClusterAPIKeyCreateRequest struct { + Name string `json:"name,omitempty"` + CrossClusterAPIKeyUpdateRequest +} + +type CrossClusterAPIKeyUpdateRequest struct { + esv1.RemoteClusterAPIKey + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +type CrossClusterAPIKeyCreateResponse struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + APIKey string `json:"api_key,omitempty"` + Encoded string `json:"encoded,omitempty"` +} + +type CrossClusterAPIKeyUpdateResponse struct { + Update string `json:"string,omitempty"` +} + +type CrossClusterAPIKeyList struct { + APIKeys []CrossClusterAPIKey `json:"api_keys,omitempty"` +} + +func (cl *CrossClusterAPIKeyList) Len() int { + if cl == nil { + return 0 + } + return len(cl.APIKeys) +} + +// GetActiveKeyWithName returns the first active key that matches the provided name or pattern. +func (cl *CrossClusterAPIKeyList) GetActiveKeyWithName(name string) *CrossClusterAPIKey { + if cl == nil || cl.Len() == 0 { + return nil + } + for _, key := range cl.APIKeys { + if key.Name == name { + return &key + } + } + return nil +} + +// KeyNames extracts the key names from a list of keys. +func (cl *CrossClusterAPIKeyList) KeyNames() sets.Set[string] { + if cl == nil || cl.Len() == 0 { + return nil + } + result := sets.New[string]() + for _, key := range cl.APIKeys { + result.Insert(key.Name) + } + return result +} + +// ForCluster returns all the API keys related to a specific client cluster. +func (cl *CrossClusterAPIKeyList) ForCluster(namespace string, name string) (*CrossClusterAPIKeyList, error) { + if cl == nil || cl.APIKeys == nil { + return nil, nil + } + crossClusterAPIKeyList := &CrossClusterAPIKeyList{ + APIKeys: make([]CrossClusterAPIKey, 0, len(cl.APIKeys)), + } + for _, apiKey := range cl.APIKeys { + elasticsearchName, err := apiKey.GetElasticsearchName() + if err != nil { + return nil, err + } + if elasticsearchName.Namespace == namespace && elasticsearchName.Name == name { + crossClusterAPIKeyList.APIKeys = append(crossClusterAPIKeyList.APIKeys, apiKey) + } + } + return crossClusterAPIKeyList, nil +} + +type CrossClusterAPIKey struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// GetElasticsearchName returns the name of the client cluster for which this key has been created. +func (c *CrossClusterAPIKey) GetElasticsearchName() (types.NamespacedName, error) { + if c == nil { + return types.NamespacedName{}, nil + } + esNameInMetadata, ok := c.Metadata["elasticsearch.k8s.elastic.co/name"] + if !ok { + return types.NamespacedName{}, fmt.Errorf("missing metadata in cross cluster API key: elasticsearch.k8s.elastic.co/name") + } + esNamespaceInMetadata, ok := c.Metadata["elasticsearch.k8s.elastic.co/namespace"] + if !ok { + return types.NamespacedName{}, fmt.Errorf("missing metadata in cross cluster API key: elasticsearch.k8s.elastic.co/namespace") + } + + namespacedName := types.NamespacedName{} + if esName, ok := esNameInMetadata.(string); ok { + namespacedName.Name = esName + } + if esNamespace, ok := esNamespaceInMetadata.(string); ok { + namespacedName.Namespace = esNamespace + } + return namespacedName, nil +} + +// RemoteClustersSettings is used to build a request to update remote clusters. +type RemoteClustersSettings struct { + PersistentSettings *SettingsGroup `json:"persistent,omitempty"` +} + +// SettingsGroup is a group of persistent settings. +type SettingsGroup struct { + Cluster RemoteClusters `json:"cluster,omitempty"` +} + +// RemoteClusters models the configuration of the remote clusters. +type RemoteClusters struct { + RemoteClusters map[string]RemoteCluster `json:"remote,omitempty"` +} + +// RemoteCluster is the set of seeds to use in a remote cluster setting. +type RemoteCluster struct { + Seeds []string `json:"seeds"` +} diff --git a/pkg/controller/elasticsearch/client/v6.go b/pkg/controller/elasticsearch/client/v6.go index 08efdc760e..2d361f073c 100644 --- a/pkg/controller/elasticsearch/client/v6.go +++ b/pkg/controller/elasticsearch/client/v6.go @@ -22,6 +22,22 @@ type clientV6 struct { baseClient } +func (c *clientV6) InvalidateCrossClusterAPIKey(context.Context, string) error { + return errNotSupportedInEs6x +} + +func (c *clientV6) CreateCrossClusterAPIKey(_ context.Context, _ CrossClusterAPIKeyCreateRequest) (CrossClusterAPIKeyCreateResponse, error) { + return CrossClusterAPIKeyCreateResponse{}, errNotSupportedInEs6x +} + +func (c *clientV6) UpdateCrossClusterAPIKey(_ context.Context, _ string, _ CrossClusterAPIKeyUpdateRequest) (CrossClusterAPIKeyUpdateResponse, error) { + return CrossClusterAPIKeyUpdateResponse{}, errNotSupportedInEs6x +} + +func (c *clientV6) GetCrossClusterAPIKeys(_ context.Context, _ string) (CrossClusterAPIKeyList, error) { + return CrossClusterAPIKeyList{}, errNotSupportedInEs6x +} + func (c *clientV6) Version() version.Version { return c.version } diff --git a/pkg/controller/elasticsearch/client/v8.go b/pkg/controller/elasticsearch/client/v8.go index 5f13e48353..fc52cd3537 100644 --- a/pkg/controller/elasticsearch/client/v8.go +++ b/pkg/controller/elasticsearch/client/v8.go @@ -35,6 +35,28 @@ func (c *clientV8) GetClusterState(ctx context.Context) (ClusterState, error) { return response, err } +func (c *clientV8) CreateCrossClusterAPIKey(ctx context.Context, request CrossClusterAPIKeyCreateRequest) (CrossClusterAPIKeyCreateResponse, error) { + var response CrossClusterAPIKeyCreateResponse + err := c.post(ctx, "/_security/cross_cluster/api_key", request, &response) + return response, err +} + +func (c *clientV8) UpdateCrossClusterAPIKey(ctx context.Context, apiKeyID string, request CrossClusterAPIKeyUpdateRequest) (CrossClusterAPIKeyUpdateResponse, error) { + var response CrossClusterAPIKeyUpdateResponse + err := c.put(ctx, fmt.Sprintf("/_security/cross_cluster/api_key/%s", apiKeyID), request, &response) + return response, err +} + +func (c *clientV8) GetCrossClusterAPIKeys(ctx context.Context, name string) (CrossClusterAPIKeyList, error) { + var response CrossClusterAPIKeyList + err := c.get(ctx, fmt.Sprintf("/_security/api_key?active_only=true&name=%s", name), &response) + return response, err +} + +func (c *clientV8) InvalidateCrossClusterAPIKey(ctx context.Context, name string) error { + return c.deleteWithObjects(ctx, "/_security/api_key", CrossClusterAPIKeyInvalidateRequest{Name: name}, nil) +} + // Equal returns true if c2 can be considered the same as c func (c *clientV8) Equal(c2 Client) bool { other, ok := c2.(*clientV8) diff --git a/pkg/controller/elasticsearch/driver/driver.go b/pkg/controller/elasticsearch/driver/driver.go index cae7fae24c..e9beaff9cd 100644 --- a/pkg/controller/elasticsearch/driver/driver.go +++ b/pkg/controller/elasticsearch/driver/driver.go @@ -12,11 +12,16 @@ import ( "time" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "k8s.io/utils/ptr" controller "sigs.k8s.io/controller-runtime/pkg/reconcile" + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/association" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common" @@ -149,6 +154,23 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { return results.WithError(err) } + // Remote Cluster Server (RCS2) Kubernetes Service reconciliation. + if d.ES.Spec.RemoteClusterServer.Enabled { + // Remote Cluster Server is enabled, ensure that the related Kubernetes Service does exist. + if _, err := common.ReconcileService(ctx, d.Client, services.NewRemoteClusterService(d.ES), &d.ES); err != nil { + results.WithError(err) + } + } else { + // Ensure that remote cluster Service does not exist. + remoteClusterService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: d.ES.Namespace, + Name: services.RemoteClusterServiceName(d.ES.Name), + }, + } + results.WithError(k8s.DeleteResourceIfExists(ctx, d.Client, remoteClusterService)) + } + resourcesState, err := reconcile.NewResourcesStateFromAPI(d.Client, d.ES) if err != nil { return results.WithError(err) @@ -325,7 +347,12 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { keystoreSecurityContext := securitycontext.For(d.Version, true) keystoreParams.SecurityContext = &keystoreSecurityContext - // setup a keystore with secure settings in an init container, if specified by the user + // Set up a keystore with secure settings in an init container, if specified by the user. + // We are also using the keystore internally for the remote cluster API keys. + remoteClusterAPIKeys, err := apiKeyStoreSecretSource(ctx, &d.ES, d.Client) + if err != nil { + return results.WithError(err) + } keystoreResources, err := keystore.ReconcileResources( ctx, d, @@ -333,6 +360,7 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { esv1.ESNamer, label.NewLabels(k8s.ExtractNamespacedName(&d.ES)), keystoreParams, + remoteClusterAPIKeys..., ) if err != nil { return results.WithError(err) @@ -373,6 +401,27 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results { return results.WithResults(d.reconcileNodeSpecs(ctx, esReachable, esClient, d.ReconcileState, *resourcesState, keystoreResources)) } +// apiKeyStoreSecretSource returns the Secret that holds the remote API keys, and which should be used as a secure settings source. +func apiKeyStoreSecretSource(ctx context.Context, es *esv1.Elasticsearch, c k8s.Client) ([]commonv1.NamespacedSecretSource, error) { + // Check if Secret exists + secretName := types.NamespacedName{ + Name: esv1.RemoteAPIKeysSecretName(es.Name), + Namespace: es.Namespace, + } + if err := c.Get(ctx, secretName, &corev1.Secret{}); err != nil { + if k8serrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return []commonv1.NamespacedSecretSource{ + { + Namespace: es.Namespace, + SecretName: secretName.Name, + }, + }, nil +} + // newElasticsearchClient creates a new Elasticsearch HTTP client for this cluster using the provided user func (d *defaultDriver) newElasticsearchClient( ctx context.Context, diff --git a/pkg/controller/elasticsearch/network/ports.go b/pkg/controller/elasticsearch/network/ports.go index 7a78b1dc3d..81340964b1 100644 --- a/pkg/controller/elasticsearch/network/ports.go +++ b/pkg/controller/elasticsearch/network/ports.go @@ -9,4 +9,6 @@ const ( HTTPPort = 9200 // TransportPort used by Elasticsearch for the Transport protocol in node to node communication TransportPort = 9300 + // RemoteClusterPort used by Elasticsearch for the remote cluster protocol. + RemoteClusterPort = 9443 ) diff --git a/pkg/controller/elasticsearch/nodespec/podspec_test.go b/pkg/controller/elasticsearch/nodespec/podspec_test.go index 0f4beb933d..0ca22b0556 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec_test.go +++ b/pkg/controller/elasticsearch/nodespec/podspec_test.go @@ -225,7 +225,7 @@ func TestBuildPodTemplateSpecWithDefaultSecurityContext(t *testing.T) { es.Spec.Version = tt.version.String() es.Spec.NodeSets[0].PodTemplate.Spec.SecurityContext = tt.userSecurityContext - cfg, err := settings.NewMergedESConfig(es.Name, tt.version, corev1.IPv4Protocol, es.Spec.HTTP, *es.Spec.NodeSets[0].Config, nil) + cfg, err := settings.NewMergedESConfig(es.Name, tt.version, corev1.IPv4Protocol, es.Spec.HTTP, *es.Spec.NodeSets[0].Config, nil, false, false) require.NoError(t, err) client := k8s.NewFakeClient(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: es.Namespace, Name: esv1.ScriptsConfigMap(es.Name)}}) @@ -308,7 +308,7 @@ func TestBuildPodTemplateSpec(t *testing.T) { ver, err := version.Parse(es.Spec.Version) require.NoError(t, err) - cfg, err := settings.NewMergedESConfig(es.Name, ver, corev1.IPv4Protocol, es.Spec.HTTP, *nodeSet.Config, tt.args.policyConfig.ElasticsearchConfig) + cfg, err := settings.NewMergedESConfig(es.Name, ver, corev1.IPv4Protocol, es.Spec.HTTP, *nodeSet.Config, tt.args.policyConfig.ElasticsearchConfig, false, false) require.NoError(t, err) actual, err := BuildPodTemplateSpec(context.Background(), tt.args.client, es, es.Spec.NodeSets[0], cfg, tt.args.keystoreResources, tt.args.setDefaultSecurityContext, tt.args.policyConfig) @@ -444,7 +444,7 @@ func Test_buildAnnotations(t *testing.T) { build() ver, err := version.Parse(sampleES.Spec.Version) require.NoError(t, err) - cfg, err := settings.NewMergedESConfig(es.Name, ver, corev1.IPv4Protocol, es.Spec.HTTP, *es.Spec.NodeSets[0].Config, nil) + cfg, err := settings.NewMergedESConfig(es.Name, ver, corev1.IPv4Protocol, es.Spec.HTTP, *es.Spec.NodeSets[0].Config, nil, false, false) require.NoError(t, err) got := buildAnnotations(es, cfg, tt.args.keystoreResources, tt.args.scriptsContent, tt.args.policyAnnotations) @@ -541,7 +541,7 @@ func Test_enableLog4JFormatMsgNoLookups(t *testing.T) { ver, err := version.Parse(sampleES.Spec.Version) require.NoError(t, err) - cfg, err := settings.NewMergedESConfig(sampleES.Name, ver, corev1.IPv4Protocol, sampleES.Spec.HTTP, *sampleES.Spec.NodeSets[0].Config, nil) + cfg, err := settings.NewMergedESConfig(sampleES.Name, ver, corev1.IPv4Protocol, sampleES.Spec.HTTP, *sampleES.Spec.NodeSets[0].Config, nil, false, false) require.NoError(t, err) client := k8s.NewFakeClient(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: sampleES.Namespace, Name: esv1.ScriptsConfigMap(sampleES.Name)}}) actual, err := BuildPodTemplateSpec(context.Background(), client, sampleES, sampleES.Spec.NodeSets[0], cfg, nil, false, PolicyConfig{}) diff --git a/pkg/controller/elasticsearch/nodespec/resources.go b/pkg/controller/elasticsearch/nodespec/resources.go index 155b589e76..40ffb8fced 100644 --- a/pkg/controller/elasticsearch/nodespec/resources.go +++ b/pkg/controller/elasticsearch/nodespec/resources.go @@ -81,7 +81,7 @@ func BuildExpectedResources( if nodeSpec.Config != nil { userCfg = *nodeSpec.Config } - cfg, err := settings.NewMergedESConfig(es.Name, ver, ipFamily, es.Spec.HTTP, userCfg, policyConfig.ElasticsearchConfig) + cfg, err := settings.NewMergedESConfig(es.Name, ver, ipFamily, es.Spec.HTTP, userCfg, policyConfig.ElasticsearchConfig, es.Spec.RemoteClusterServer.Enabled, es.HasRemoteClusterAPIKey()) if err != nil { return nil, err } diff --git a/pkg/controller/elasticsearch/nodespec/statefulset.go b/pkg/controller/elasticsearch/nodespec/statefulset.go index 6d9d0eda9f..9a5cca01f2 100644 --- a/pkg/controller/elasticsearch/nodespec/statefulset.go +++ b/pkg/controller/elasticsearch/nodespec/statefulset.go @@ -18,6 +18,7 @@ import ( sset "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/statefulset" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/network" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/settings" es_sset "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/sset" esvolume "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/volume" @@ -33,6 +34,22 @@ func HeadlessServiceName(ssetName string) string { // HeadlessService returns a headless service for the given StatefulSet func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { nsn := k8s.ExtractNamespacedName(es) + ports := []corev1.ServicePort{ + { + Name: es.Spec.HTTP.Protocol(), + Protocol: corev1.ProtocolTCP, + Port: network.HTTPPort, + }, + } + if es.Spec.RemoteClusterServer.Enabled { + ports = append(ports, + corev1.ServicePort{ + Name: services.RemoteClusterServicePortName, + Protocol: corev1.ProtocolTCP, + Port: network.RemoteClusterPort, + }, + ) + } return corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -44,13 +61,7 @@ func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { Type: corev1.ServiceTypeClusterIP, ClusterIP: corev1.ClusterIPNone, Selector: label.NewStatefulSetLabels(nsn, ssetName), - Ports: []corev1.ServicePort{ - { - Name: es.Spec.HTTP.Protocol(), - Protocol: corev1.ProtocolTCP, - Port: network.HTTPPort, - }, - }, + Ports: ports, // allow nodes to discover themselves via DNS while they are booting up ie. are not ready yet PublishNotReadyAddresses: true, }, diff --git a/pkg/controller/elasticsearch/remotecluster/elasticsearch.go b/pkg/controller/elasticsearch/remotecluster/elasticsearch.go index ffdcf8683c..7a46ba48f8 100644 --- a/pkg/controller/elasticsearch/remotecluster/elasticsearch.go +++ b/pkg/controller/elasticsearch/remotecluster/elasticsearch.go @@ -115,6 +115,11 @@ func updateSettingsInternal( remoteClustersToUpdate = append(remoteClustersToUpdate, name) // Declare remote cluster in ES seedHosts := []string{services.ExternalTransportServiceHost(remoteCluster.ElasticsearchRef.NamespacedName())} + if remoteCluster.APIKey != nil { + // User specified an API key. It means that the remote cluster is expected to be accessed using the remote + // cluster Service instead of relying on the transport layer. + seedHosts = []string{services.RemoteClusterServerServiceHost(remoteCluster.ElasticsearchRef.NamespacedName())} + } remoteClustersToApply[name] = esclient.RemoteCluster{Seeds: seedHosts} // Ensure this cluster is tracked in the annotation remoteClustersInAnnotation[name] = struct{}{} diff --git a/pkg/controller/elasticsearch/services/services.go b/pkg/controller/elasticsearch/services/services.go index 71284a22bb..c3fb559b27 100644 --- a/pkg/controller/elasticsearch/services/services.go +++ b/pkg/controller/elasticsearch/services/services.go @@ -5,7 +5,6 @@ package services import ( - "context" "fmt" "math/rand" "strconv" @@ -25,6 +24,8 @@ import ( const ( globalServiceSuffix = ".svc" + + RemoteClusterServicePortName = "rcs" ) // TransportServiceName returns the name for the transport service associated to this cluster @@ -75,11 +76,22 @@ func InternalServiceName(esName string) string { return esv1.InternalHTTPService(esName) } +// RemoteClusterServiceName returns the name for the remote cluster service used when the cluster is expected to be accessed +// using the remote cluster server. Managed by the operator exclusively. +func RemoteClusterServiceName(esName string) string { + return esv1.RemoteClusterService(esName) +} + // ExternalTransportServiceHost returns the hostname and the port used to reach Elasticsearch's transport endpoint. func ExternalTransportServiceHost(es types.NamespacedName) string { return stringsutil.Concat(TransportServiceName(es.Name), ".", es.Namespace, globalServiceSuffix, ":", strconv.Itoa(network.TransportPort)) } +// RemoteClusterServerServiceHost returns the hostname and the port used to reach Elasticsearch's remote cluster server endpoint. +func RemoteClusterServerServiceHost(es types.NamespacedName) string { + return stringsutil.Concat(RemoteClusterServiceName(es.Name), ".", es.Namespace, globalServiceSuffix, ":", strconv.Itoa(network.RemoteClusterPort)) +} + // ExternalServiceURL returns the URL used to reach Elasticsearch's external endpoint. func ExternalServiceURL(es esv1.Elasticsearch) string { return stringsutil.Concat(es.Spec.HTTP.Protocol(), "://", ExternalServiceName(es.Name), ".", es.Namespace, globalServiceSuffix, ":", strconv.Itoa(network.HTTPPort)) @@ -141,45 +153,30 @@ func NewInternalService(es esv1.Elasticsearch) *corev1.Service { } } -// IsServiceReady checks if a service has one or more ready endpoints. -func IsServiceReady(c k8s.Client, service corev1.Service) (bool, error) { - endpoints := corev1.Endpoints{} - namespacedName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} - - if err := c.Get(context.Background(), namespacedName, &endpoints); err != nil { - return false, err - } - for _, subs := range endpoints.Subsets { - if len(subs.Addresses) > 0 { - return true, nil - } - } - return false, nil -} - -// GetExternalService returns the external service associated to the given Elasticsearch cluster. -func GetExternalService(c k8s.Client, es esv1.Elasticsearch) (corev1.Service, error) { - return getServiceByName(c, es, ExternalServiceName(es.Name)) -} - -// GetInternalService returns the internally managed service associated to the given Elasticsearch cluster. -func GetInternalService(c k8s.Client, es esv1.Elasticsearch) (corev1.Service, error) { - return getServiceByName(c, es, InternalServiceName(es.Name)) -} - -func getServiceByName(c k8s.Client, es esv1.Elasticsearch, serviceName string) (corev1.Service, error) { - var svc corev1.Service - - namespacedName := types.NamespacedName{ - Namespace: es.Namespace, - Name: serviceName, +// NewRemoteClusterService returns the service associated to the remote cluster service for the given cluster. +func NewRemoteClusterService(es esv1.Elasticsearch) *corev1.Service { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: RemoteClusterServiceName(es.Name), + Namespace: es.Namespace, + Labels: label.NewLabels(k8s.ExtractNamespacedName(&es)), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: label.NewLabels(k8s.ExtractNamespacedName(&es)), + PublishNotReadyAddresses: true, + ClusterIP: "None", + }, } - - if err := c.Get(context.Background(), namespacedName, &svc); err != nil { - return corev1.Service{}, err + labels := label.NewLabels(k8s.ExtractNamespacedName(&es)) + ports := []corev1.ServicePort{ + { + Name: RemoteClusterServicePortName, + Protocol: corev1.ProtocolTCP, + Port: network.RemoteClusterPort, + }, } - - return svc, nil + return defaults.SetServiceDefaults(svc, labels, labels, ports) } type urlProvider struct { diff --git a/pkg/controller/elasticsearch/settings/merged_config.go b/pkg/controller/elasticsearch/settings/merged_config.go index d02e12a103..067ec67a3e 100644 --- a/pkg/controller/elasticsearch/settings/merged_config.go +++ b/pkg/controller/elasticsearch/settings/merged_config.go @@ -34,15 +34,16 @@ func NewMergedESConfig( httpConfig commonv1.HTTPConfig, userConfig commonv1.Config, esConfigFromStackConfigPolicy *common.CanonicalConfig, + remoteClusterServerEnabled, remoteClusterClientEnabled bool, ) (CanonicalConfig, error) { userCfg, err := common.NewCanonicalConfigFrom(userConfig.Data) if err != nil { return CanonicalConfig{}, err } - config := baseConfig(clusterName, ver, ipFamily).CanonicalConfig + config := baseConfig(clusterName, ver, ipFamily, remoteClusterServerEnabled).CanonicalConfig err = config.MergeWith( - xpackConfig(ver, httpConfig).CanonicalConfig, + xpackConfig(ver, httpConfig, remoteClusterServerEnabled, remoteClusterClientEnabled).CanonicalConfig, userCfg, esConfigFromStackConfigPolicy, ) @@ -53,7 +54,7 @@ func NewMergedESConfig( } // baseConfig returns the base ES configuration to apply for the given cluster -func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamily) *CanonicalConfig { +func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamily, remoteClusterServerEnabled bool) *CanonicalConfig { cfg := map[string]interface{}{ // derive node name dynamically from the pod name, injected as env var esv1.NodeName: "${" + EnvPodName + "}", @@ -72,6 +73,12 @@ func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamil esv1.PathLogs: volume.ElasticsearchLogsMountPath, } + if remoteClusterServerEnabled { + cfg[esv1.RemoteClusterEnabled] = "true" + cfg[esv1.RemoteClusterPublishHost] = "${" + EnvPodName + "}.${" + HeadlessServiceName + "}.${" + EnvNamespace + "}.svc" + cfg[esv1.RemoteClusterHost] = "0" + } + // seed hosts setting name changed starting ES 7.X fileProvider := "file" if ver.Major < 7 { @@ -91,7 +98,7 @@ func baseConfig(clusterName string, ver version.Version, ipFamily corev1.IPFamil } // xpackConfig returns the configuration bit related to XPack settings -func xpackConfig(ver version.Version, httpCfg commonv1.HTTPConfig) *CanonicalConfig { +func xpackConfig(ver version.Version, httpCfg commonv1.HTTPConfig, remoteClusterServerEnabled, remoteClusterClientEnabled bool) *CanonicalConfig { // enable x-pack security, including TLS cfg := map[string]interface{}{ // x-pack security general settings @@ -121,6 +128,30 @@ func xpackConfig(ver version.Version, httpCfg commonv1.HTTPConfig) *CanonicalCon esv1.XPackSecurityHttpSslCertificateAuthorities: path.Join(volume.HTTPCertificatesSecretVolumeMountPath, certificates.CAFileName), } + if remoteClusterServerEnabled { + cfg[esv1.XPackSecurityRemoteClusterServerSslKey] = path.Join( + volume.TransportCertificatesSecretVolumeMountPath, + "${POD_NAME}."+certificates.KeyFileName, + ) + cfg[esv1.XPackSecurityRemoteClusterServerSslCertificate] = path.Join( + volume.TransportCertificatesSecretVolumeMountPath, + "${POD_NAME}."+certificates.CertFileName, + ) + cfg[esv1.XPackSecurityRemoteClusterServerSslCertificateAuthorities] = []string{ + path.Join(volume.TransportCertificatesSecretVolumeMountPath, certificates.CAFileName), + path.Join(volume.RemoteCertificateAuthoritiesSecretVolumeMountPath, certificates.CAFileName), + } + } + + if remoteClusterClientEnabled { + cfg[esv1.XPackSecurityRemoteClusterClientSslKey] = true + cfg[esv1.XPackSecurityRemoteClusterClientSslCertificateAuthorities] = []string{ + // Include /usr/share/elasticsearch/config/transport-certs/ca.crt to trust any additional CA in transport.tls.certificateAuthorities + path.Join(volume.TransportCertificatesSecretVolumeMountPath, certificates.CAFileName), + path.Join(volume.RemoteCertificateAuthoritiesSecretVolumeMountPath, certificates.CAFileName), + } + } + // always enable the built-in file and native internal realms for user auth, ordered as first if ver.Major < 7 { // 6.x syntax diff --git a/pkg/controller/elasticsearch/settings/merged_config_test.go b/pkg/controller/elasticsearch/settings/merged_config_test.go index 848d504074..7651650e4b 100644 --- a/pkg/controller/elasticsearch/settings/merged_config_test.go +++ b/pkg/controller/elasticsearch/settings/merged_config_test.go @@ -42,13 +42,81 @@ func TestNewMergedESConfig(t *testing.T) { }) tests := []struct { - name string - version string - ipFamily corev1.IPFamily - cfgData map[string]interface{} - policyCfgData *common.CanonicalConfig - assert func(cfg CanonicalConfig) + name string + version string + ipFamily corev1.IPFamily + remoteClusterServerEnabled bool + remoteClusterClientEnabled bool + cfgData map[string]interface{} + policyCfgData *common.CanonicalConfig + assert func(cfg CanonicalConfig) }{ + { + name: "No remote cluster client or server by default", + version: "8.15.0", + ipFamily: corev1.IPv4Protocol, + cfgData: map[string]interface{}{}, + assert: func(cfg CanonicalConfig) { + // Remote cluster client configuration. + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + // Remote cluster server configuration. + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.key"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate_authorities"}))) + }, + }, + { + name: "Remote cluster client is enabled", + version: "8.15.0", + ipFamily: corev1.IPv4Protocol, + remoteClusterClientEnabled: true, + cfgData: map[string]interface{}{}, + assert: func(cfg CanonicalConfig) { + // Remote cluster client configuration. + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + // Remote cluster server configuration. + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.key"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate_authorities"}))) + }, + }, + { + name: "Remote cluster server is enabled", + version: "8.15.0", + ipFamily: corev1.IPv4Protocol, + remoteClusterServerEnabled: true, + cfgData: map[string]interface{}{}, + assert: func(cfg CanonicalConfig) { + // Remote cluster client configuration. + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + require.Equal(t, 0, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_client.ssl.enabled"}))) + // Remote cluster server configuration. + require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster_server.enabled"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster.publish_host"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"remote_cluster.host"}))) + // Remote cluster server TLS configuration. + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.key"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate"}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{"xpack.security.remote_cluster_server.ssl.certificate_authorities"}))) + }, + }, + { + name: "in 6.x, empty config should have the default file and native realm settings configured", + version: "6.8.0", + ipFamily: corev1.IPv4Protocol, + cfgData: map[string]interface{}{}, + assert: func(cfg CanonicalConfig) { + require.Equal(t, 0, len(cfg.HasKeys([]string{nodeML}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.XPackSecurityAuthcRealmsFile1Type}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.XPackSecurityAuthcRealmsFile1Order}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.XPackSecurityAuthcRealmsNative1Type}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.XPackSecurityAuthcRealmsNative1Order}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{esv1.ShardAwarenessAttributes}))) + require.Equal(t, 1, len(cfg.HasKeys([]string{nodeAttrNodeName}))) + }, + }, { name: "in 6.x, empty config should have the default file and native realm settings configured", version: "6.8.0", @@ -259,7 +327,7 @@ func TestNewMergedESConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ver, err := version.Parse(tt.version) require.NoError(t, err) - cfg, err := NewMergedESConfig("clusterName", ver, tt.ipFamily, commonv1.HTTPConfig{}, commonv1.Config{Data: tt.cfgData}, tt.policyCfgData) + cfg, err := NewMergedESConfig("clusterName", ver, tt.ipFamily, commonv1.HTTPConfig{}, commonv1.Config{Data: tt.cfgData}, tt.policyCfgData, tt.remoteClusterServerEnabled, tt.remoteClusterClientEnabled) require.NoError(t, err) tt.assert(cfg) }) diff --git a/pkg/controller/elasticsearch/validation/validations.go b/pkg/controller/elasticsearch/validation/validations.go index d53ea67304..04a20e02ef 100644 --- a/pkg/controller/elasticsearch/validation/validations.go +++ b/pkg/controller/elasticsearch/validation/validations.go @@ -73,6 +73,7 @@ func validations(ctx context.Context, checker license.Checker, exposedNodeLabels validPVCNaming, validMonitoring, validAssociations, + supportsRemoteClusterUsingAPIKey, func(proposed esv1.Elasticsearch) field.ErrorList { return validLicenseLevel(ctx, proposed, checker) }, @@ -134,6 +135,37 @@ func supportedVersion(es esv1.Elasticsearch) field.ErrorList { return field.ErrorList{field.Invalid(field.NewPath("spec").Child("version"), es.Spec.Version, unsupportedVersionMsg)} } +func supportsRemoteClusterUsingAPIKey(es esv1.Elasticsearch) field.ErrorList { + ver, err := version.Parse(es.Spec.Version) + if err != nil { + return field.ErrorList{field.Invalid(field.NewPath("spec").Child("version"), es.Spec.Version, parseVersionErrMsg)} + } + var errs field.ErrorList + if es.Spec.RemoteClusterServer.Enabled && ver.LE(esv1.RemoteClusterAPIKeysMinVersion) { + errs = append(errs, field.Invalid( + field.NewPath("spec").Child("remoteClusterServer"), + es.Spec.Version, + fmt.Sprintf( + "minimum required version for remote cluster server is %s but desired version is %s", + esv1.RemoteClusterAPIKeysMinVersion, + es.Spec.Version, + ), + )) + } + if es.HasRemoteClusterAPIKey() && ver.LE(esv1.RemoteClusterAPIKeysMinVersion) { + errs = append(errs, field.Invalid( + field.NewPath("spec").Child("remoteClusters").Child("*").Key("apiKey"), + es.Spec.Version, + fmt.Sprintf( + "minimum required version for remote cluster using API keys is %s but desired version is %s", + esv1.RemoteClusterAPIKeysMinVersion, + es.Spec.Version, + ), + )) + } + return errs +} + // hasCorrectNodeRoles checks whether Elasticsearch node roles are correctly configured. // The rules are: // There must be at least one master node. diff --git a/pkg/controller/elasticsearch/validation/validations_test.go b/pkg/controller/elasticsearch/validation/validations_test.go index ea8d7935df..0e1ce64376 100644 --- a/pkg/controller/elasticsearch/validation/validations_test.go +++ b/pkg/controller/elasticsearch/validation/validations_test.go @@ -203,6 +203,104 @@ func Test_supportedVersion(t *testing.T) { } } +func Test_supportsRemoteClusterUsingAPIKey(t *testing.T) { + tests := []struct { + name string + es esv1.Elasticsearch + expectErrors bool + }{ + { + name: "no remote cluster settings that relies on API keys", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + }, + Spec: esv1.ElasticsearchSpec{Version: "7.0.0"}, + }, + expectErrors: false, + }, + { + name: "some remote cluster API keys before required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.9.99", + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "bar", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + }, + }, + }, + expectErrors: true, + }, + { + name: "some remote cluster API keys with min required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.10.0", + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "bar", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + }, + }, + }, + expectErrors: false, + }, + { + name: "remote cluster without API keys before required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.9.99", + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "bar", + }, + }, + }, + }, + expectErrors: false, + }, + { + name: "remote cluster server enabled with min required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.10.0", + RemoteClusterServer: esv1.RemoteClusterServer{Enabled: true}, + }, + }, + expectErrors: false, + }, + { + name: "remote cluster server enabled before min required version", + es: esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Spec: esv1.ElasticsearchSpec{ + Version: "8.9.99", + RemoteClusterServer: esv1.RemoteClusterServer{Enabled: true}, + }, + }, + expectErrors: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := supportsRemoteClusterUsingAPIKey(tt.es) + actualErrors := len(actual) > 0 + if tt.expectErrors != actualErrors { + t.Errorf("failed supportsRemoteClusterUsingAPIKey(). Name: %v, actual %v, wanted: %v, value: %v", tt.name, actual, tt.expectErrors, tt.es.Spec.Version) + } + }) + } +} + func Test_validName(t *testing.T) { tests := []struct { name string diff --git a/pkg/controller/remoteca/controller.go b/pkg/controller/remoteca/controller.go deleted file mode 100644 index b60337b06d..0000000000 --- a/pkg/controller/remoteca/controller.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package remoteca - -import ( - "context" - "fmt" - "time" - - "go.elastic.co/apm/v2" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" - esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/association" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/license" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/operator" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/tracing" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/watches" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/certificates/remoteca" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" - "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" - ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" - "github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac" -) - -const ( - name = "remoteca-controller" - - EventReasonClusterCaCertNotFound = "ClusterCaCertNotFound" -) - -var ( - defaultRequeue = reconcile.Result{Requeue: true, RequeueAfter: 20 * time.Second} -) - -// Add creates a new RemoteCa Controller and adds it to the manager with default RBAC. -func Add(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operator.Parameters) error { - r := NewReconciler(mgr, accessReviewer, params) - c, err := common.NewController(mgr, name, r, params) - if err != nil { - return err - } - return addWatches(mgr, c, r) -} - -// NewReconciler returns a new reconcile.Reconciler -func NewReconciler(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operator.Parameters) *ReconcileRemoteCa { - c := mgr.GetClient() - return &ReconcileRemoteCa{ - Client: c, - accessReviewer: accessReviewer, - watches: watches.NewDynamicWatches(), - recorder: mgr.GetEventRecorderFor(name), - licenseChecker: license.NewLicenseChecker(c, params.OperatorNamespace), - Parameters: params, - } -} - -var _ reconcile.Reconciler = &ReconcileRemoteCa{} - -// ReconcileRemoteCa reconciles remote CA Secrets. -type ReconcileRemoteCa struct { - k8s.Client - operator.Parameters - accessReviewer rbac.AccessReviewer - recorder record.EventRecorder - watches watches.DynamicWatches - licenseChecker license.Checker - - // iteration is the number of times this controller has run its Reconcile method - iteration uint64 -} - -// Reconcile reads that state of the cluster for the expected remote clusters in this Kubernetes cluster. -// It copies the remote CA Secrets so they can be trusted by every peer Elasticsearch clusters. -func (r *ReconcileRemoteCa) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - ctx = common.NewReconciliationContext(ctx, &r.iteration, r.Tracer, name, "es_name", request) - defer common.LogReconciliationRun(ulog.FromContext(ctx))() - defer tracing.EndContextTransaction(ctx) - - // Fetch the local Elasticsearch spec - es := esv1.Elasticsearch{} - err := r.Get(ctx, request.NamespacedName, &es) - if err != nil { - if errors.IsNotFound(err) { - return deleteAllRemoteCa(ctx, r, request.NamespacedName) - } - return reconcile.Result{}, err - } - - if common.IsUnmanaged(ctx, &es) { - ulog.FromContext(ctx).Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", es.Namespace, "es_name", es.Name) - return reconcile.Result{}, nil - } - - return doReconcile(ctx, r, &es) -} - -// deleteAllRemoteCa deletes all associated remote certificate authorities -func deleteAllRemoteCa(ctx context.Context, r *ReconcileRemoteCa, es types.NamespacedName) (reconcile.Result, error) { - span, _ := apm.StartSpan(ctx, "delete_all_remote_ca", tracing.SpanTypeApp) - defer span.End() - - remoteClusters, err := remoteClustersInvolvedWith(ctx, r.Client, es) - if err != nil { - return reconcile.Result{}, err - } - results := &reconciler.Results{} - for remoteCluster := range remoteClusters { - if err := deleteCertificateAuthorities(ctx, r, es, remoteCluster); err != nil { - results.WithError(err) - } - } - return results.Aggregate() -} - -func doReconcile( - ctx context.Context, - r *ReconcileRemoteCa, - localEs *esv1.Elasticsearch, -) (reconcile.Result, error) { - log := ulog.FromContext(ctx) - - localClusterKey := k8s.ExtractNamespacedName(localEs) - - expectedRemoteClusters, err := getExpectedRemoteClusters(ctx, r.Client, localEs) - if err != nil { - return reconcile.Result{}, err - } - - enabled, err := r.licenseChecker.EnterpriseFeaturesEnabled(ctx) - if err != nil { - return defaultRequeue, err - } - if !enabled && len(expectedRemoteClusters) > 0 { - log.V(1).Info( - "Remote cluster controller is an enterprise feature. Enterprise features are disabled", - "namespace", localEs.Namespace, "es_name", localEs.Name, - ) - return reconcile.Result{}, nil - } - - // Get all the clusters to which this reconciled cluster is connected to according to the existing remote CAs. - // remoteClustersInvolved is used to delete the CA certificates and cancel any trust relationships - // that may have existed in the past but should not exist anymore. - remoteClustersInvolved, err := remoteClustersInvolvedWith(ctx, r.Client, localClusterKey) - if err != nil { - return reconcile.Result{}, err - } - - results := &reconciler.Results{} - // Create or update expected remote CA - for remoteEsKey := range expectedRemoteClusters { - // Get the remote Elasticsearch cluster associated with this remote CA - remoteEs := &esv1.Elasticsearch{} - if err := r.Client.Get(ctx, remoteEsKey, remoteEs); err != nil { - if errors.IsNotFound(err) { - // Remote cluster does not exist, skip it - continue - } - return reconcile.Result{}, err - } - accessAllowed, err := isRemoteClusterAssociationAllowed(ctx, r.accessReviewer, localEs, remoteEs, r.recorder) - if err != nil { - return reconcile.Result{}, err - } - // if the remote CA exists but isn't allowed anymore, it will be deleted next - if !accessAllowed { - continue - } - delete(remoteClustersInvolved, remoteEsKey) - results.WithResults(createOrUpdateCertificateAuthorities(ctx, r, localEs, remoteEs)) - if results.HasError() { - return results.Aggregate() - } - } - - // Delete existing but not expected remote CA - for toDelete := range remoteClustersInvolved { - log.V(1).Info("Deleting remote CA", - "local_namespace", localEs.Namespace, - "local_name", localEs.Name, - "remote_namespace", toDelete.Namespace, - "remote_name", toDelete.Name, - ) - results.WithError(deleteCertificateAuthorities(ctx, r, localClusterKey, toDelete)) - } - return results.WithResult(association.RequeueRbacCheck(r.accessReviewer)).Aggregate() -} - -func caCertMissingError(cluster types.NamespacedName) string { - return fmt.Sprintf("Cannot find CA certificate cluster %s/%s", cluster.Namespace, cluster.Name) -} - -// getExpectedRemoteClusters returns all the remote cluster keys for which a remote ca should created -// The CA certificates must be copied from the remote cluster to the local one and vice versa -func getExpectedRemoteClusters( - ctx context.Context, - c k8s.Client, - associatedEs *esv1.Elasticsearch, -) (map[types.NamespacedName]struct{}, error) { - span, _ := apm.StartSpan(ctx, "get_expected_remote_ca", tracing.SpanTypeApp) - defer span.End() - expectedRemoteClusters := make(map[types.NamespacedName]struct{}) - - // Add remote clusters declared in the Spec - for _, remoteCluster := range associatedEs.Spec.RemoteClusters { - if !remoteCluster.ElasticsearchRef.IsDefined() { - continue - } - esRef := remoteCluster.ElasticsearchRef.WithDefaultNamespace(associatedEs.Namespace) - expectedRemoteClusters[esRef.NamespacedName()] = struct{}{} - } - - var list esv1.ElasticsearchList - if err := c.List(ctx, &list, &client.ListOptions{}); err != nil { - return nil, err - } - - // Seek for Elasticsearch resources where this cluster is declared as a remote cluster - for _, es := range list.Items { - es := es - for _, remoteCluster := range es.Spec.RemoteClusters { - if !remoteCluster.ElasticsearchRef.IsDefined() { - continue - } - esRef := remoteCluster.ElasticsearchRef.WithDefaultNamespace(es.Namespace) - if esRef.Namespace == associatedEs.Namespace && - esRef.Name == associatedEs.Name { - expectedRemoteClusters[k8s.ExtractNamespacedName(&es)] = struct{}{} - } - } - } - - return expectedRemoteClusters, nil -} - -// remoteClustersInvolvedWith returns for a given Elasticsearch cluster all the Elasticsearch keys for which -// the remote certificate authorities have been copied, i.e. all the other Elasticsearch clusters for which this cluster -// has been involved in a remote cluster association. -// In order to get all of them we: -// 1. List all the remote CA copied locally. -// 2. List all the other Elasticsearch clusters for which the CA of the given cluster has been copied. -func remoteClustersInvolvedWith( - ctx context.Context, - c k8s.Client, - es types.NamespacedName, -) (map[types.NamespacedName]struct{}, error) { - span, _ := apm.StartSpan(ctx, "get_current_remote_ca", tracing.SpanTypeApp) - defer span.End() - - currentRemoteClusters := make(map[types.NamespacedName]struct{}) - - // 1. Get clusters whose CA has been copied into the local namespace. - var remoteCAList corev1.SecretList - if err := c.List(ctx, - &remoteCAList, - client.InNamespace(es.Namespace), - remoteca.Labels(es.Name), - ); err != nil { - return nil, err - } - for _, remoteCA := range remoteCAList.Items { - remoteNs := remoteCA.Labels[RemoteClusterNamespaceLabelName] - remoteEs := remoteCA.Labels[RemoteClusterNameLabelName] - currentRemoteClusters[types.NamespacedName{ - Namespace: remoteNs, - Name: remoteEs, - }] = struct{}{} - } - - // 2. Get clusters for which the CA of the local cluster has been copied. - if err := c.List(ctx, - &remoteCAList, - client.MatchingLabels(map[string]string{ - commonv1.TypeLabelName: remoteca.TypeLabelValue, - RemoteClusterNamespaceLabelName: es.Namespace, - RemoteClusterNameLabelName: es.Name, - }), - ); err != nil { - return nil, err - } - for _, remoteCA := range remoteCAList.Items { - remoteEs := remoteCA.Labels[label.ClusterNameLabelName] - currentRemoteClusters[types.NamespacedName{ - Namespace: remoteCA.Namespace, - Name: remoteEs, - }] = struct{}{} - } - - return currentRemoteClusters, nil -} diff --git a/pkg/controller/remoteca/controller_test.go b/pkg/controller/remoteca/controller_test.go deleted file mode 100644 index 7e00af14be..0000000000 --- a/pkg/controller/remoteca/controller_test.go +++ /dev/null @@ -1,456 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package remoteca - -import ( - "context" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" - esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/license" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/watches" - "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/certificates/transport" - "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" - "github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac" -) - -type clusterBuilder struct { - name, namespace string - remoteClusters []commonv1.ObjectSelector -} - -func newClusteBuilder(namespace, name string) *clusterBuilder { - return &clusterBuilder{ - name: name, - namespace: namespace, - } -} - -func (cb *clusterBuilder) withRemoteCluster(namespace, name string) *clusterBuilder { - cb.remoteClusters = append(cb.remoteClusters, commonv1.ObjectSelector{ - Name: name, - Namespace: namespace, - }) - return cb -} - -func (cb *clusterBuilder) build() *esv1.Elasticsearch { - remoteClusters := make([]esv1.RemoteCluster, len(cb.remoteClusters)) - for i, remoteCluster := range cb.remoteClusters { - remoteClusters[i] = esv1.RemoteCluster{ - ElasticsearchRef: commonv1.LocalObjectSelector{ - Name: remoteCluster.Name, - Namespace: remoteCluster.Namespace, - }} - } - - return &esv1.Elasticsearch{ - ObjectMeta: v1.ObjectMeta{ - Namespace: cb.namespace, - Name: cb.name, - }, - Spec: esv1.ElasticsearchSpec{ - RemoteClusters: remoteClusters, - }, - } -} - -type fakeAccessReviewer struct { - allowed bool - err error -} - -func (f *fakeAccessReviewer) AccessAllowed(_ context.Context, _ string, _ string, _ runtime.Object) (bool, error) { - return f.allowed, f.err -} - -func fakePublicCa(namespace, name string) *corev1.Secret { - namespacedName := types.NamespacedName{ - Name: name, - Namespace: namespace, - } - transportPublicCertKey := transport.PublicCertsSecretRef(namespacedName) - return &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Namespace: transportPublicCertKey.Namespace, - Name: transportPublicCertKey.Name, - }, - Data: map[string][]byte{ - certificates.CAFileName: []byte(namespacedName.String()), - }, - } -} - -// remoteCa builds an expected remote Ca -func remoteCa(localNamespace, localName, remoteNamespace, remoteName string) *corev1.Secret { - remoteNamespacedName := types.NamespacedName{ - Name: remoteName, - Namespace: remoteNamespace, - } - return &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Namespace: localNamespace, - Name: remoteCASecretName(localName, remoteNamespacedName), - Labels: map[string]string{ - "common.k8s.elastic.co/type": "remote-ca", - "elasticsearch.k8s.elastic.co/cluster-name": localName, - "elasticsearch.k8s.elastic.co/remote-cluster-name": remoteName, - "elasticsearch.k8s.elastic.co/remote-cluster-namespace": remoteNamespace, - }, - }, - Data: map[string][]byte{ - certificates.CAFileName: []byte(remoteNamespacedName.String()), - }, - } -} - -func withDataCert(caSecret *corev1.Secret, newCa []byte) *corev1.Secret { - caSecret.Data[certificates.CAFileName] = newCa - return caSecret -} - -func TestReconcileRemoteCa_Reconcile(t *testing.T) { - type fields struct { - clusters []client.Object - accessReviewer rbac.AccessReviewer - licenseChecker license.Checker - } - type args struct { - request reconcile.Request - } - tests := []struct { - name string - fields fields - args args - - expectedSecrets []*corev1.Secret - unexpectedSecrets []types.NamespacedName - want reconcile.Result - wantErr bool - }{ - { - name: "Simple remote cluster ns1/es1 -> ns2/es2", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - expectedSecrets: []*corev1.Secret{ - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "Bi-directional remote cluster ns1/es1 <-> ns2/es2", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").withRemoteCluster("ns1", "es1").build(), - fakePublicCa("ns2", "es2"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - expectedSecrets: []*corev1.Secret{ - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "Deleted remote cluster", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - unexpectedSecrets: []types.NamespacedName{ - { - Namespace: "ns1", - Name: remoteCASecretName("es1", types.NamespacedName{ - Namespace: "ns2", - Name: "es2", - }), - }, - { - Namespace: "ns2", - Name: remoteCASecretName("es2", types.NamespacedName{ - Namespace: "ns1", - Name: "es1", - }), - }, - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "CA content has been updated, remote ca must be reconciled", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - withDataCert(remoteCa("ns1", "es1", "ns2", "es2"), []byte("foo")), - withDataCert(remoteCa("ns2", "es2", "ns1", "es1"), []byte("bar")), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - expectedSecrets: []*corev1.Secret{ - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - // ns1/es1 has been deleted - all related secrets in other namespaces must be deleted - name: "Deleted cluster", - fields: fields{ - clusters: []client.Object{ - // ns2/es2 - newClusteBuilder("ns2", "es2").withRemoteCluster("ns1", "es1").build(), - fakePublicCa("ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - // ns3/es3 - newClusteBuilder("ns3", "es3").withRemoteCluster("ns1", "es1").build(), - fakePublicCa("ns3", "es3"), - remoteCa("ns3", "es3", "ns1", "es1"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - unexpectedSecrets: []types.NamespacedName{ - { - Namespace: "ns3", - Name: remoteCASecretName("es3", types.NamespacedName{ - Namespace: "ns1", - Name: "es1", - }), - }, - { - Namespace: "ns2", - Name: remoteCASecretName("es2", types.NamespacedName{ - Namespace: "ns1", - Name: "es1", - }), - }, - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "No enterprise license, remote ca are not created", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: false}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - unexpectedSecrets: []types.NamespacedName{ - { - Namespace: "ns1", - Name: remoteCASecretName("es1", types.NamespacedName{ - Namespace: "ns2", - Name: "es2", - }), - }, - { - Namespace: "ns2", - Name: remoteCASecretName("es2", types.NamespacedName{ - Namespace: "ns1", - Name: "es1", - }), - }, - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "No enterprise license, existing remote ca are left untouched", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - accessReviewer: &fakeAccessReviewer{allowed: true}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: false}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - expectedSecrets: []*corev1.Secret{ - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - want: reconcile.Result{}, - wantErr: false, - }, - { - name: "Association is not allowed, existing remote ca are removed", - fields: fields{ - clusters: []client.Object{ - newClusteBuilder("ns1", "es1").withRemoteCluster("ns2", "es2").build(), - fakePublicCa("ns1", "es1"), - newClusteBuilder("ns2", "es2").build(), - fakePublicCa("ns2", "es2"), - remoteCa("ns1", "es1", "ns2", "es2"), - remoteCa("ns2", "es2", "ns1", "es1"), - }, - accessReviewer: &fakeAccessReviewer{allowed: false}, - licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, - }, - args: args{ - request: reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: "es1", - Namespace: "ns1", - }, - }, - }, - unexpectedSecrets: []types.NamespacedName{ - { - Namespace: "ns1", - Name: remoteCASecretName("es1", types.NamespacedName{Namespace: "ns2", Name: "es2"}), - }, - { - Namespace: "ns2", - Name: remoteCASecretName("es2", types.NamespacedName{Namespace: "ns1", Name: "es1"}), - }, - }, - want: reconcile.Result{}, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - w := watches.NewDynamicWatches() - r := &ReconcileRemoteCa{ - Client: k8s.NewFakeClient(tt.fields.clusters...), - accessReviewer: tt.fields.accessReviewer, - watches: w, - licenseChecker: tt.fields.licenseChecker, - recorder: record.NewFakeRecorder(10), - } - got, err := r.Reconcile(context.Background(), tt.args.request) - if (err != nil) != tt.wantErr { - t.Errorf("ReconcileRemoteCa.Reconcile() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ReconcileRemoteCa.Reconcile() = %v, want %v", got, tt.want) - } - // Check that expected secrets are here - for _, expectedSecret := range tt.expectedSecrets { - var actualSecret corev1.Secret - assert.NoError(t, r.Client.Get(context.Background(), types.NamespacedName{Namespace: expectedSecret.Namespace, Name: expectedSecret.Name}, &actualSecret)) - // Compare content - actualCa, ok := actualSecret.Data[certificates.CAFileName] - assert.True(t, ok) - assert.Equal(t, expectedSecret.Data[certificates.CAFileName], actualCa) - // Compare labels - assert.NotNil(t, actualSecret.Labels) - assert.Equal(t, expectedSecret.Labels, actualSecret.Labels) - } - // Check that unexpected secrets does not exist - for _, unexpectedSecret := range tt.unexpectedSecrets { - var actualSecret corev1.Secret - err := r.Client.Get(context.Background(), types.NamespacedName{Namespace: unexpectedSecret.Namespace, Name: unexpectedSecret.Name}, &actualSecret) - assert.True(t, apierrors.IsNotFound(err)) - } - }) - } -} diff --git a/pkg/controller/remotecluster/apikey.go b/pkg/controller/remotecluster/apikey.go new file mode 100644 index 0000000000..d6be1c8601 --- /dev/null +++ b/pkg/controller/remotecluster/apikey.go @@ -0,0 +1,193 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/hash" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" + esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" +) + +// reconcileAPIKeys creates or updates the API Keys for the remote/client cluster, +// which may have several references (.Spec.RemoteClusters) to the cluster being reconciled. +func reconcileAPIKeys( + ctx context.Context, + c k8s.Client, + activeAPIKeys esclient.CrossClusterAPIKeyList, // all the API Keys in the reconciled/local cluster + remoteServerES *esv1.Elasticsearch, // the Elasticsearch cluster being reconciled, where the API keys must be created/invalidated + remoteClientES *esv1.Elasticsearch, // the remote Elasticsearch cluster which is going to act as the client, where the API keys are going to be stored in the keystore Secret + remoteClusterRefs []esv1.RemoteCluster, // the expected API keys for that client cluster + esClient esclient.Client, // ES client for the reconciled cluster which is going to act as the server + keystoreProvider *keystore.Provider, +) *reconciler.Results { + log := ulog.FromContext(ctx).WithValues( + "remote_server_namespace", remoteServerES.Namespace, + "remote_server_name", remoteServerES.Name, + "remote_client_namespace", remoteClientES.Namespace, + "remote_client_name", remoteClientES.Name, + ) + results := &reconciler.Results{} + // clientClusterAPIKeyStore is used to reconcile encoded API keys in the client cluster, to inject new API keys + // or to delete the ones which are no longer needed. + clientClusterAPIKeyStore, err := keystoreProvider.ForCluster(ctx, log, remoteClientES) + if err != nil { + return results.WithError(err) + } + + // Maintain a list of the expected API keys for that specific client cluster, to detect the ones which are no longer expected in the reconciled cluster. + expectedKeysInRemoteServerES := sets.New[string]() + // Same for the aliases + expectedAliases := sets.New[string]() + activeAPIKeysNames := activeAPIKeys.KeyNames() + for _, remoteClusterRef := range remoteClusterRefs { + apiKeyName := fmt.Sprintf("eck-%s-%s-%s", remoteClientES.Namespace, remoteClientES.Name, remoteClusterRef.Name) + expectedKeysInRemoteServerES.Insert(apiKeyName) + expectedAliases.Insert(remoteClusterRef.Name) + if remoteClusterRef.APIKey == nil { + if activeAPIKeysNames.Has(apiKeyName) { + // We found an API key for that client cluster while it is not expected to have one. + // It may happen when the user switched back from API keys to the legacy remote cluster. + log.Info("Invalidating API key as remote cluster is not configured to use it", "alias", remoteClusterRef.Name) + if err := esClient.InvalidateCrossClusterAPIKey(ctx, apiKeyName); err != nil { + return results.WithError(err) + } + } + continue + } + + // Attempt to get an existing API Key with that key name. + activeAPIKey := activeAPIKeys.GetActiveKeyWithName(apiKeyName) + expectedHash := hash.HashObject(remoteClusterRef.APIKey) + if activeAPIKey == nil { + if err := createAPIKey(ctx, log, remoteClusterRef, apiKeyName, esClient, remoteClientES, expectedHash, clientClusterAPIKeyStore, remoteServerES); err != nil { + return results.WithError(err) + } + } else { + // If an API key already exists ensure that the access field is the expected one using the hash + if err := maybeUpdateAPIKey(ctx, log, esClient, clientClusterAPIKeyStore, remoteClusterRef, activeAPIKey, apiKeyName, remoteClientES, expectedHash); err != nil { + return results.WithError(err) + } + } + } + + // Get all the active API keys which have been created for that client cluster. + activeAPIKeysForClientCluster, err := activeAPIKeys.ForCluster(remoteClientES.Namespace, remoteClientES.Name) + if err != nil { + return results.WithError(err) + } + // Invalidate all the keys related to that local cluster which are not expected. + for keyName := range activeAPIKeysForClientCluster.KeyNames() { + if !expectedKeysInRemoteServerES.Has(keyName) { + // Unexpected key, let's invalidate it. + log.Info("Invalidating unexpected API key", "key", keyName) + if err := esClient.InvalidateCrossClusterAPIKey(ctx, keyName); err != nil { + return results.WithError(err) + } + } + } + + // Delete all the keys in the keystore which are not expected. + aliases := clientClusterAPIKeyStore.ForCluster(remoteServerES.Namespace, remoteServerES.Name) + for existingAlias := range aliases { + if expectedAliases.Has(existingAlias) { + continue + } + clientClusterAPIKeyStore.Delete(existingAlias) + } + + // Save the generated keys in the keystore. + if err := clientClusterAPIKeyStore.Save(ctx, c, remoteClientES); err != nil { + return err + } + return nil +} + +func createAPIKey( + ctx context.Context, + log logr.Logger, + remoteCluster esv1.RemoteCluster, + apiKeyName string, + esClient esclient.Client, + clientES *esv1.Elasticsearch, + expectedHash string, + clientClusterAPIKeyStore *keystore.APIKeyStore, + reconciledES *esv1.Elasticsearch, +) error { + // Active API key not found, let's create a new one. + log.Info("Creating API key", "alias", remoteCluster.Name, "key", apiKeyName) + apiKey, err := esClient.CreateCrossClusterAPIKey(ctx, esclient.CrossClusterAPIKeyCreateRequest{ + Name: apiKeyName, + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + RemoteClusterAPIKey: *remoteCluster.APIKey, + Metadata: newMetadataFor(clientES, expectedHash), + }, + }) + if err != nil { + return err + } + clientClusterAPIKeyStore.Update(reconciledES.Name, reconciledES.Namespace, remoteCluster.Name, apiKey.ID, apiKey.Encoded) + return nil +} + +func maybeUpdateAPIKey( + ctx context.Context, + log logr.Logger, + esClient esclient.Client, + clientClusterAPIKeyStore *keystore.APIKeyStore, + remoteCluster esv1.RemoteCluster, + activeAPIKey *esclient.CrossClusterAPIKey, + apiKeyName string, + clientES *esv1.Elasticsearch, + expectedHash string, +) error { + // Ensure that the API key is in the keystore + if clientClusterAPIKeyStore.KeyIDFor(remoteCluster.Name) != activeAPIKey.ID { + // We have a problem here, the API Key ID in Elasticsearch does not match the API Key recorded in the Secret. + // Invalidate the API Key in ES and requeue + log.Info("Invalidating API key as it does not match the one in keystore", "alias", remoteCluster.Name, "key", apiKeyName) + if err := esClient.InvalidateCrossClusterAPIKey(ctx, activeAPIKey.Name); err != nil { + return err + } + return fmt.Errorf( + "cluster key id for alias %s %s (%s), does not match the one stored in the keystore of %s/%s", + remoteCluster.Name, activeAPIKey.Name, activeAPIKey.ID, clientES.Namespace, clientES.Name, + ) + } + currentHash := activeAPIKey.Metadata["elasticsearch.k8s.elastic.co/config-hash"] + if currentHash != expectedHash { + log.Info("Updating API key", "alias", remoteCluster.Name) + // Update the Key + _, err := esClient.UpdateCrossClusterAPIKey(ctx, activeAPIKey.ID, esclient.CrossClusterAPIKeyUpdateRequest{ + RemoteClusterAPIKey: *remoteCluster.APIKey, + Metadata: newMetadataFor(clientES, expectedHash), + }) + if err != nil { + return err + } + } + return nil +} + +// newMetadataFor returns the metadata to be set in the Elasticsearch API keys metadata in the Elasticsearch cluster +// state, not on a Kubernetes object. +func newMetadataFor(clientES *esv1.Elasticsearch, expectedHash string) map[string]interface{} { + return map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": expectedHash, + "elasticsearch.k8s.elastic.co/name": clientES.Name, + "elasticsearch.k8s.elastic.co/namespace": clientES.Namespace, + "elasticsearch.k8s.elastic.co/uid": clientES.UID, + "elasticsearch.k8s.elastic.co/managed-by": "eck", + } +} diff --git a/pkg/controller/remotecluster/controller.go b/pkg/controller/remotecluster/controller.go new file mode 100644 index 0000000000..067a4e1d78 --- /dev/null +++ b/pkg/controller/remotecluster/controller.go @@ -0,0 +1,449 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "fmt" + "time" + + "go.elastic.co/apm/v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/association" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common" + commonesclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/esclient" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/license" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/operator" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/tracing" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/watches" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/certificates/remoteca" + esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac" +) + +const ( + name = "remotecluster-controller" + + EventReasonClusterCaCertNotFound = "ClusterCaCertNotFound" +) + +var ( + defaultRequeue = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} +) + +// Add creates a new ReconcileRemoteClusters Controller and adds it to the manager with default RBAC. +func Add(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operator.Parameters) error { + r := NewReconciler(mgr, accessReviewer, params) + c, err := common.NewController(mgr, name, r, params) + if err != nil { + return err + } + return addWatches(mgr, c, r) +} + +// NewReconciler returns a new reconcile.Reconciler +func NewReconciler(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operator.Parameters) *ReconcileRemoteClusters { + c := mgr.GetClient() + return &ReconcileRemoteClusters{ + Client: c, + accessReviewer: accessReviewer, + keystoreProvider: keystore.NewProvider(c), + watches: watches.NewDynamicWatches(), + recorder: mgr.GetEventRecorderFor(name), + licenseChecker: license.NewLicenseChecker(c, params.OperatorNamespace), + Parameters: params, + esClientProvider: commonesclient.NewClient, + } +} + +var _ reconcile.Reconciler = &ReconcileRemoteClusters{} + +// ReconcileRemoteClusters reconciles remote clusters Secrets and API Keys. +type ReconcileRemoteClusters struct { + k8s.Client + operator.Parameters + accessReviewer rbac.AccessReviewer + recorder record.EventRecorder + watches watches.DynamicWatches + licenseChecker license.Checker + esClientProvider commonesclient.Provider + keystoreProvider *keystore.Provider + + // iteration is the number of times this controller has run its Reconcile method + iteration uint64 +} + +// Reconcile reads that state of the cluster for the expected remote clusters in this Kubernetes cluster. +// It copies the remote CA Secrets so they can be trusted by every peer Elasticsearch clusters. +func (r *ReconcileRemoteClusters) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + ctx = common.NewReconciliationContext(ctx, &r.iteration, r.Tracer, name, "es_name", request) + defer common.LogReconciliationRun(ulog.FromContext(ctx))() + defer tracing.EndContextTransaction(ctx) + + // Fetch the local Elasticsearch spec + es := esv1.Elasticsearch{} + err := r.Get(ctx, request.NamespacedName, &es) + if err != nil { + if errors.IsNotFound(err) { + r.keystoreProvider.ForgetCluster(request.NamespacedName) + return deleteAllRemoteCa(ctx, r, request.NamespacedName) + } + return reconcile.Result{}, err + } + + if common.IsUnmanaged(ctx, &es) { + ulog.FromContext(ctx).Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", es.Namespace, "es_name", es.Name) + return reconcile.Result{}, nil + } + return doReconcile(ctx, r, &es) +} + +// deleteAllRemoteCa deletes all associated remote certificate authorities +func deleteAllRemoteCa(ctx context.Context, r *ReconcileRemoteClusters, es types.NamespacedName) (reconcile.Result, error) { + span, _ := apm.StartSpan(ctx, "delete_all_remote_ca", tracing.SpanTypeApp) + defer span.End() + + associatedCAs, err := getAssociatedRemoteCAs(ctx, r.Client, es) + if err != nil { + return reconcile.Result{}, err + } + results := &reconciler.Results{} + for remoteCluster := range associatedCAs { + if err := deleteCertificateAuthorities(ctx, r, es, remoteCluster); err != nil { + results.WithError(err) + } + } + return results.Aggregate() +} + +func doReconcile( + ctx context.Context, + r *ReconcileRemoteClusters, + remoteServer *esv1.Elasticsearch, +) (reconcile.Result, error) { + log := ulog.FromContext(ctx) + + remoteServerKey := k8s.ExtractNamespacedName(remoteServer) + + expectedRemoteClients, err := getExpectedRemoteClientsFor(ctx, r.Client, remoteServer) + if err != nil { + return reconcile.Result{}, err + } + + enabled, err := r.licenseChecker.EnterpriseFeaturesEnabled(ctx) + if err != nil { + return defaultRequeue, err + } + if !enabled && len(expectedRemoteClients) > 0 { + log.V(1).Info( + "Remote cluster controller is an enterprise feature. Enterprise features are disabled", + "namespace", remoteServer.Namespace, "es_name", remoteServer.Name, + ) + return reconcile.Result{}, nil + } + + // Get all the clusters to which this reconciled cluster is connected to according to the existing remote CAs. + // associatedRemoteCAs is used to delete the CA certificates and cancel any trust relationships + // that may have existed in the past but should not exist anymore. + associatedRemoteCAs, err := getAssociatedRemoteCAs(ctx, r.Client, remoteServerKey) + if err != nil { + return reconcile.Result{}, err + } + + var ( + activeAPIKeys esclient.CrossClusterAPIKeyList + esClient esclient.Client + ) + remoteServerSupportsClusterAPIKeys, err := remoteServer.SupportsRemoteClusterAPIKeys() + if err != nil { + return reconcile.Result{}, err + } + results := &reconciler.Results{} + if remoteServerSupportsClusterAPIKeys.IsTrue() { + // Check if the ES API is available. We need it to create, update and invalidate + // API keys in this cluster. + if !services.NewElasticsearchURLProvider(*remoteServer, r.Client).HasEndpoints() { + log.Info("Elasticsearch API is not available yet") + return results.WithResult(defaultRequeue).Aggregate() + } + // Create a new client + newEsClient, err := r.esClientProvider(ctx, r.Client, r.Dialer, *remoteServer) + if err != nil { + return reconcile.Result{}, err + } + // Check that the API is available + esClient = newEsClient + // Get all the API Keys, for that specific client, on the reconciled cluster. + crossClusterAPIKeys, err := esClient.GetCrossClusterAPIKeys(ctx, "eck-*") + if err != nil { + return reconcile.Result{}, err + } + activeAPIKeys = crossClusterAPIKeys + } + + // apiKeyReconciledRemoteClients is used to track all the client clusters for which API keys have already been reconciled. + // This is used to garbage collect API keys for clusters which have been deleted and are not in expectedRemoteClusters. + apiKeyReconciledRemoteClients := sets.New[types.NamespacedName]() + + // Main loop to: + // 1. Create or update expected remote CA. + // 2. Create or update API keys and keystores. + for remoteClientKey, remoteClusterRefs := range expectedRemoteClients { + // Get the remote/client Elasticsearch cluster associated with this local/reconciled cluster. + remoteClient := &esv1.Elasticsearch{} + if err := r.Client.Get(ctx, remoteClientKey, remoteClient); err != nil { + if errors.IsNotFound(err) { + // Remote client cluster does not exist, invalidate API keys for that client cluster. + apiKeyReconciledRemoteClients.Insert(remoteClientKey) + results.WithResults(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) + continue + } + return reconcile.Result{}, err + } + log := log.WithValues( + "remote_server_namespace", remoteServer.Namespace, + "remote_server", remoteServer.Name, + "remote_client_namespace", remoteClient.Namespace, + "remote_client_name", remoteClient.Name, + ) + accessAllowed, err := isRemoteClusterAssociationAllowed(ctx, r.accessReviewer, remoteServer, remoteClient, r.recorder) + if err != nil { + return reconcile.Result{}, err + } + // if the remote CA exists but isn't allowed anymore, it will be deleted next + if !accessAllowed { + // Remove from the expected remote cluster to clean up local keystore. + delete(expectedRemoteClients, remoteClientKey) + // Invalidate API keys for that client cluster. + apiKeyReconciledRemoteClients.Insert(remoteClientKey) + results.WithResults(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, nil, esClient, r.keystoreProvider)) + continue + } + delete(associatedRemoteCAs, remoteClientKey) + results.WithResults(createOrUpdateCertificateAuthorities(ctx, r, remoteServer, remoteClient)) + if results.HasError() { + return results.Aggregate() + } + + // RCS2, first check that both the reconciled and the client clusters are compatible. + clientClusterSupportsClusterAPIKeys, err := remoteClient.SupportsRemoteClusterAPIKeys() + if err != nil { + results.WithError(err) + continue + } + + if !clientClusterSupportsClusterAPIKeys.IsSet() { + log.Info("Client cluster version is not available in status yet, skipping API keys reconciliation") + continue + } + + if !remoteServerSupportsClusterAPIKeys.IsSet() { + log.Info("Cluster version is not available in status yet, skipping API keys reconciliation") + continue + } + + if clientClusterSupportsClusterAPIKeys.IsFalse() && remoteServerSupportsClusterAPIKeys.IsTrue() { + err := fmt.Errorf("client cluster %s/%s is running version %s which does not support remote cluster keys", remoteClient.Namespace, remoteClient.Name, remoteClient.Spec.Version) + log.Error(err, "cannot configure remote cluster settings") + continue + } + // Reconcile the API Keys. + apiKeyReconciledRemoteClients.Insert(remoteClientKey) + results.WithResults(reconcileAPIKeys(ctx, r.Client, activeAPIKeys, remoteServer, remoteClient, remoteClusterRefs, esClient, r.keystoreProvider)) + } + + if remoteServerSupportsClusterAPIKeys.IsTrue() { + // ************************************************************** + // Delete orphaned API keys from clusters which have been deleted + // ************************************************************** + for _, activeAPIKey := range activeAPIKeys.APIKeys { + clientCluster, err := activeAPIKey.GetElasticsearchName() + if err != nil { + results.WithError(err) + continue + } + if _, exists := apiKeyReconciledRemoteClients[clientCluster]; exists { + // API keys for that client cluster have already been reconciled, skip. + continue + } + // This API key in the local cluster state belongs to an unknown cluster which is not expected and has not been reconciled. + log.Info(fmt.Sprintf("Invalidating API key %s which belongs to unknown cluster %s", activeAPIKey.Name, clientCluster)) + results.WithError(esClient.InvalidateCrossClusterAPIKey(ctx, activeAPIKey.Name)) + } + + // ********************************************* + // Delete unexpected keys in the local keystore. + // ********************************************* + expectedAliases := expectedAliases(remoteServer, expectedRemoteClients) + apiKeyStore, err := r.keystoreProvider.ForCluster(ctx, log, remoteServer) + if err != nil { + return results.WithError(err).Aggregate() + } + + for alias := range apiKeyStore.GetAliases() { + if expectedAliases.Has(alias) { + // Expected alias + continue + } + // Unexpected + log.Info(fmt.Sprintf("Removing unexpected remote API key %s", alias)) + apiKeyStore.Delete(alias) + } + results.WithResults(apiKeyStore.Save(ctx, r.Client, remoteServer)) + } + + // Delete existing but not expected remote CA + for toDelete := range associatedRemoteCAs { + log.V(1).Info("Deleting remote CA", + "local_namespace", remoteServer.Namespace, + "local_name", remoteServer.Name, + "remote_namespace", toDelete.Namespace, + "remote_name", toDelete.Name, + ) + results.WithError(deleteCertificateAuthorities(ctx, r, remoteServerKey, toDelete)) + } + return results.WithResult(association.RequeueRbacCheck(r.accessReviewer)).Aggregate() +} + +func expectedAliases( + localCluster *esv1.Elasticsearch, + expectedRemoteCluster map[types.NamespacedName][]esv1.RemoteCluster, +) sets.Set[string] { + aliases := sets.New[string]() + for _, remoteCluster := range localCluster.Spec.RemoteClusters { + clientClusterNamespacedName := remoteCluster.ElasticsearchRef.WithDefaultNamespace(localCluster.Namespace).NamespacedName() + if _, ok := expectedRemoteCluster[clientClusterNamespacedName]; !ok { + // Not expected, might have been filtered by RBAC rules + continue + } + if remoteCluster.APIKey == nil { + // Not using remote cluster server. + continue + } + aliases.Insert(remoteCluster.Name) + } + return aliases +} + +func caCertMissingError(cluster types.NamespacedName) string { + return fmt.Sprintf("Cannot find CA certificate cluster %s/%s", cluster.Namespace, cluster.Name) +} + +// getExpectedRemoteClientsFor returns all the remote cluster keys for which a remote ca and an API Key should be created. +// The CA certificates must be copied from the remote cluster to the local one and vice versa. +// The API Key is created in the remote cluster and injected in the keystore of the local cluster. +func getExpectedRemoteClientsFor( + ctx context.Context, + c k8s.Client, + associatedEs *esv1.Elasticsearch, +) (map[types.NamespacedName][]esv1.RemoteCluster, error) { + span, _ := apm.StartSpan(ctx, "get_expected_remote_clusters", tracing.SpanTypeApp) + defer span.End() + expectedRemoteClusters := make(map[types.NamespacedName][]esv1.RemoteCluster) + + // AddKey remote clusters declared in the Spec + for _, remoteCluster := range associatedEs.Spec.RemoteClusters { + if !remoteCluster.ElasticsearchRef.IsDefined() { + continue + } + esRef := remoteCluster.ElasticsearchRef.WithDefaultNamespace(associatedEs.Namespace) + expectedRemoteClusters[esRef.NamespacedName()] = nil + } + + var list esv1.ElasticsearchList + if err := c.List(ctx, &list, &client.ListOptions{}); err != nil { + return nil, err + } + + // Seek for Elasticsearch resources where this cluster is declared as a remote cluster + for _, es := range list.Items { + es := es + for _, remoteCluster := range es.Spec.RemoteClusters { + if !remoteCluster.ElasticsearchRef.IsDefined() { + continue + } + esRef := remoteCluster.ElasticsearchRef.WithDefaultNamespace(es.Namespace) + if esRef.Namespace == associatedEs.Namespace && + esRef.Name == associatedEs.Name { + clientClusterName := k8s.ExtractNamespacedName(&es) + expectedRemoteClusters[clientClusterName] = append(expectedRemoteClusters[clientClusterName], remoteCluster) + } + } + } + + return expectedRemoteClusters, nil +} + +// getAssociatedRemoteCAs returns for a given Elasticsearch cluster all the Elasticsearch keys for which +// the remote certificate authorities have been copied, i.e. all the other Elasticsearch clusters for which this cluster +// has been involved in a remote cluster association. +// In order to get all of them we: +// 1. List all the remote CA copied locally. +// 2. List all the other Elasticsearch clusters for which the CA of the given cluster has been copied. +func getAssociatedRemoteCAs( + ctx context.Context, + c k8s.Client, + es types.NamespacedName, +) (map[types.NamespacedName]struct{}, error) { + span, _ := apm.StartSpan(ctx, "get_current_remote_ca", tracing.SpanTypeApp) + defer span.End() + + currentRemoteClusters := make(map[types.NamespacedName]struct{}) + + // 1. Get clusters whose CA has been copied into the local namespace. + var remoteCAList corev1.SecretList + if err := c.List(ctx, + &remoteCAList, + client.InNamespace(es.Namespace), + remoteca.Labels(es.Name), + ); err != nil { + return nil, err + } + for _, remoteCA := range remoteCAList.Items { + remoteNs := remoteCA.Labels[RemoteClusterNamespaceLabelName] + remoteEs := remoteCA.Labels[RemoteClusterNameLabelName] + currentRemoteClusters[types.NamespacedName{ + Namespace: remoteNs, + Name: remoteEs, + }] = struct{}{} + } + + // 2. Get clusters for which the CA of the local cluster has been copied. + if err := c.List(ctx, + &remoteCAList, + client.MatchingLabels(map[string]string{ + commonv1.TypeLabelName: remoteca.TypeLabelValue, + RemoteClusterNamespaceLabelName: es.Namespace, + RemoteClusterNameLabelName: es.Name, + }), + ); err != nil { + return nil, err + } + for _, remoteCA := range remoteCAList.Items { + remoteEs := remoteCA.Labels[label.ClusterNameLabelName] + currentRemoteClusters[types.NamespacedName{ + Namespace: remoteCA.Namespace, + Name: remoteEs, + }] = struct{}{} + } + + return currentRemoteClusters, nil +} diff --git a/pkg/controller/remotecluster/controller_test.go b/pkg/controller/remotecluster/controller_test.go new file mode 100644 index 0000000000..5ee2fa5e11 --- /dev/null +++ b/pkg/controller/remotecluster/controller_test.go @@ -0,0 +1,975 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "reflect" + "slices" + "testing" + + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/license" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/watches" + esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/net" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac" +) + +func TestRemoteCluster_Reconcile(t *testing.T) { + type fields struct { + clusters []client.Object + accessReviewer rbac.AccessReviewer + licenseChecker license.Checker + } + type args struct { + request reconcile.Request + } + type wantEsAPICalls struct { + getCrossClusterAPIKeys []string + invalidateCrossClusterAPIKey []string + crossClusterAPIKeyCreateRequests []esclient.CrossClusterAPIKeyCreateRequest + updateCrossClusterAPIKey map[string]esclient.CrossClusterAPIKeyUpdateRequest + } + tests := []struct { + name string + fields fields + args args + + expectedCASecrets []*corev1.Secret + unexpectedSecrets []types.NamespacedName + want reconcile.Result + wantErr bool + + // API keys fake client and expected resources. + fakeESClient *fakeESClient + wantEsAPICalls wantEsAPICalls + expectedKeystoreSecrets []*corev1.Secret + }{ + { + name: "Simple remote cluster ns1/es1 -> ns2/es2, ns1", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "[No API Keys] Bi-directional remote cluster ns1/es1 <-> ns2/es2", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").withRemoteCluster("ns1", "es1").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + // The test below simulates the following situation: + // * ns1/es1 is the cluster reconciled. + // * ns1/es2 is the client cluster. + name: "With API Keys, simple topology", + fields: fields{ + clusters: slices.Concat( + // Clusters + newClusterBuilder("ns1", "es1", "8.15.0").build(), + newClusterBuilder("ns1", "es2", "8.15.0"). + // ns1/es2 -> ns1/es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + []client.Object{ + // Certificates + fakePublicCa("ns1", "es1"), + fakePublicCa("ns1", "es2"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + // reconciled cluster is es1 + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns1", "es2"), + remoteCa("ns1", "es2", "ns1", "es1"), + }, + wantEsAPICalls: wantEsAPICalls{ + getCrossClusterAPIKeys: []string{"eck-*"}, + crossClusterAPIKeyCreateRequests: []esclient.CrossClusterAPIKeyCreateRequest{ + { + // ns1/es1 is expected to create an API key for ns1/es2 + Name: "eck-ns1-es2-generated-alias-from-ns1-es2-to-ns1-es1-with-api-key", + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es2", + "elasticsearch.k8s.elastic.co/namespace": "ns1", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + }, + }, + expectedKeystoreSecrets: []*corev1.Secret{ + { + // Keystore for es2 must be updated. + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns1-es2-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns1-es2-generated-alias-from-ns1-es2-to-ns1-es1-with-api-key"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns1", + Name: "es2-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.generated-alias-from-ns1-es2-to-ns1-es1-with-api-key.credentials": []byte("generated-encoded-key-from-fake-es-client-for-eck-ns1-es2-generated-alias-from-ns1-es2-to-ns1-es1-with-api-key"), + }, + }, + }, + }, + { + // The test below simulates the following situation: + // * ns1/es1 is the cluster reconciled. + // * There are 3 remote cluster accessing ns1/es1: + // * ns2/es2: new cluster, API key must be created + // * ns3/es3: new cluster, API key must be created + // * ns4/es4: existing remote cluster: one key must be updated, the other one must be deleted. + // * ns/es5: this cluster no long exists, key must be deleted. + name: "With API Keys, complex topology", + fields: fields{ + clusters: slices.Concat( + // Clusters + newClusterBuilder("ns1", "es1", "8.15.0"). + // es1 -> es2 + withAPIKey("ns2", "es2", &esv1.RemoteClusterAPIKey{}). + // es1 -> es3 + withAPIKey("ns3", "es3", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns2", "es2", "8.15.0"). + // es2 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns3", "es3", "8.15.0"). + // es3 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns4", "es4", "8.15.0"). + // es4-> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + []client.Object{ + // Certificates + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + fakePublicCa("ns3", "es3"), + fakePublicCa("ns4", "es4"), + // Assume that ns2/es2 has already a key in its keystore, the goal is to test that the existing keys are not altered. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns2", + Name: "es2-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + // Keystore for ns4/es4, key already exists, this Secret is not expected to be modified. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns4-es4-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es4", + }, + Namespace: "ns4", + Name: "es4-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.generated-alias-from-ns4-es4-to-ns1-es1-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + // Assume that ns1/es1 keystore already exists, unexpected keys should be removed. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns1-es1-to-ns2-es2-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-alias-from-ns1-es1-to-ns3-es3-with-api-key":{"namespace":"ns3","name":"es3","id":"apikey-to-es3"},"api-key-to-non-existent-alias":{"namespace":"nsx","name":"esx","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es1", + }, + Namespace: "ns1", + Name: "es1-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.generated-alias-from-ns1-es1-to-ns2-es2-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns3-es3-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + // The key below should be removed + "cluster.remote.api-key-to-non-existent-alias.credentials": []byte("encoded-key-for-api-for-non-existent-cluster"), + }, + }, + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + // reconciled cluster is es1 + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + fakeESClient: &fakeESClient{ + existingCrossClusterAPIKeys: esclient.CrossClusterAPIKeyList{ + // API key for es4 already exists but with a wrong hash we should expect an update. + APIKeys: []esclient.CrossClusterAPIKey{ + { + ID: "generated-id-from-fake-es-client-eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key", + Name: "eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + // The key below belongs to a cluster which no longer exists, it must be invalidated. + { + ID: "apikey-from-es5-to-es1", + Name: "eck-ns5-es5-generated-ns1-es1-0-with-api-key", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es5", + "elasticsearch.k8s.elastic.co/namespace": "ns5", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + // The key below belongs to the existing cluster es4 which is no longer referencing es1 using that alias, it must be invalidated. + { + ID: "apikey-from-es4-to-es1-old-alias", + Name: "eck-ns4-es4-to-ns1-es1-0-old-alias", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + }, + }, + wantEsAPICalls: wantEsAPICalls{ + getCrossClusterAPIKeys: []string{"eck-*"}, + invalidateCrossClusterAPIKey: []string{"eck-ns4-es4-to-ns1-es1-0-old-alias", "eck-ns5-es5-generated-ns1-es1-0-with-api-key"}, + updateCrossClusterAPIKey: map[string]esclient.CrossClusterAPIKeyUpdateRequest{ + "generated-id-from-fake-es-client-eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key": { + RemoteClusterAPIKey: esv1.RemoteClusterAPIKey{}, + Metadata: map[string]any{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + // We expect 2 keys to be created for ns1/es1 + crossClusterAPIKeyCreateRequests: []esclient.CrossClusterAPIKeyCreateRequest{ + { + Name: "eck-ns2-es2-generated-alias-from-ns2-es2-to-ns1-es1-with-api-key", + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es2", + "elasticsearch.k8s.elastic.co/namespace": "ns2", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + { + Name: "eck-ns3-es3-generated-alias-from-ns3-es3-to-ns1-es1-with-api-key", + CrossClusterAPIKeyUpdateRequest: esclient.CrossClusterAPIKeyUpdateRequest{ + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es3", + "elasticsearch.k8s.elastic.co/namespace": "ns3", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + }, + }, + expectedKeystoreSecrets: []*corev1.Secret{ + // Unexpected keys in es1 keystore must be removed. + { + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns1-es1-to-ns2-es2-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-alias-from-ns1-es1-to-ns3-es3-with-api-key":{"namespace":"ns3","name":"es3","id":"apikey-to-es3"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es1", + }, + Namespace: "ns1", + Name: "es1-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.generated-alias-from-ns1-es1-to-ns2-es2-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns3-es3-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + { + // Keystore for es2 must be updated. + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"},"generated-alias-from-ns2-es2-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns2-es2-generated-alias-from-ns2-es2-to-ns1-es1-with-api-key"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns2", + Name: "es2-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns2-es2-to-ns1-es1-with-api-key.credentials": []byte("generated-encoded-key-from-fake-es-client-for-eck-ns2-es2-generated-alias-from-ns2-es2-to-ns1-es1-with-api-key"), + }, + }, + { + // Keystore for es3 must be created. + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns3-es3-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns3-es3-generated-alias-from-ns3-es3-to-ns1-es1-with-api-key"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es3", + }, + Namespace: "ns3", + Name: "es3-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.generated-alias-from-ns3-es3-to-ns1-es1-with-api-key.credentials": []byte("generated-encoded-key-from-fake-es-client-for-eck-ns3-es3-generated-alias-from-ns3-es3-to-ns1-es1-with-api-key"), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns4-es4-to-ns1-es1-with-api-key":{"namespace":"ns1","name":"es1","id":"generated-id-from-fake-es-client-eck-ns4-es4-generated-alias-from-ns4-es4-to-ns1-es1-with-api-key"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es4", + }, + Namespace: "ns4", + Name: "es4-es-remote-api-keys", + }, + Data: map[string][]byte{ + "cluster.remote.generated-alias-from-ns4-es4-to-ns1-es1-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + }, + }, + { + // Same test as above but the associations are no longer permitted. + // * ns1/es1 is the cluster reconciled. + // * There are 3 remote cluster accessing ns1/es1: + // * ns2/es2: new cluster, API key must not be created + // * ns3/es3: new cluster, API key must not be created + // * ns4/es4: existing remote cluster: 2 keys must be deleted. + // * ns/es5: this cluster no long exists, key must be deleted. + name: "With API Keys: associations are no longer permitted", + fields: fields{ + clusters: slices.Concat( + // Clusters + newClusterBuilder("ns1", "es1", "8.15.0"). + // es1 -> es2 + withAPIKey("ns2", "es2", &esv1.RemoteClusterAPIKey{}). + // es1 -> es3 + withAPIKey("ns3", "es3", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns2", "es2", "8.15.0"). + // es2 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns3", "es3", "8.15.0"). + // es3 -> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + newClusterBuilder("ns4", "es4", "8.15.0"). + // es4-> es1 + withAPIKey("ns1", "es1", &esv1.RemoteClusterAPIKey{}). + build(), + []client.Object{ + // Certificates + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + fakePublicCa("ns3", "es3"), + fakePublicCa("ns4", "es4"), + // Assume that ns2/es2 has already a key in its keystore, the goal is to test that the existing keys are not altered. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns2", + Name: "es2-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + // Keystore for ns4/es4, key already exists, this Secret must be deleted since association is no longer permitted. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-ns1-es1-0-with-api-key":{"namespace":"ns1","name":"es1","id":"apikey-from-es4-to-es1"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es4", + }, + Namespace: "ns4", + Name: "es4-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.generated-ns1-es1-0-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + // Assume that ns1/es1 keystore already exists, unexpected keys should be removed. + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"generated-alias-from-ns1-es1-to-ns2-es2-with-api-key":{"namespace":"ns2","name":"es2","id":"apikey-to-es2"},"generated-alias-from-ns1-es1-to-ns3-es3-with-api-key":{"namespace":"ns3","name":"es3","id":"apikey-to-es3"},"api-key-to-non-existent-alias":{"namespace":"nsx","name":"esx","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es1", + }, + Namespace: "ns1", + Name: "es1-es-remote-api-keys", + UID: uuid.NewUUID(), + }, + Data: map[string][]byte{ + "cluster.remote.generated-alias-from-ns1-es1-to-ns2-es2-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + "cluster.remote.generated-alias-from-ns1-es1-to-ns3-es3-with-api-key.credentials": []byte("encoded-key-for-existing-api-key"), + // The key below should be removed + "cluster.remote.api-key-to-non-existent-alias.credentials": []byte("encoded-key-for-api-for-non-existent-cluster"), + }, + }, + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: false}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + // reconciled cluster is es1 + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + fakeESClient: &fakeESClient{ + existingCrossClusterAPIKeys: esclient.CrossClusterAPIKeyList{ + // API key for es4 already exists but with a wrong hash we should expect an update. + APIKeys: []esclient.CrossClusterAPIKey{ + { + ID: "apikey-from-es4-to-es1", + Name: "eck-ns4-es4-generated-ns1-es1-0-with-api-key", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + // The key below belongs to a cluster which no longer exists, it must be invalidated. + { + ID: "apikey-from-es5-to-es1", + Name: "eck-ns5-es5-generated-ns1-es1-0-with-api-key", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "1384987056", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es5", + "elasticsearch.k8s.elastic.co/namespace": "ns5", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + // The key below belongs to the existing cluster es4 which is no longer referencing es1 using that alias, it must be invalidated. + { + ID: "apikey-from-es4-to-es1-old-alias", + Name: "eck-ns4-es4-to-ns1-es1-0-old-alias", + Metadata: map[string]interface{}{ + "elasticsearch.k8s.elastic.co/config-hash": "unexpected-hash", + "elasticsearch.k8s.elastic.co/managed-by": "eck", + "elasticsearch.k8s.elastic.co/name": "es4", + "elasticsearch.k8s.elastic.co/namespace": "ns4", + "elasticsearch.k8s.elastic.co/uid": types.UID(""), + }, + }, + }, + }, + }, + wantEsAPICalls: wantEsAPICalls{ + getCrossClusterAPIKeys: []string{"eck-*"}, + invalidateCrossClusterAPIKey: []string{"eck-ns4-es4-generated-ns1-es1-0-with-api-key", "eck-ns4-es4-to-ns1-es1-0-old-alias", "eck-ns5-es5-generated-ns1-es1-0-with-api-key"}, + // No update allowed. + updateCrossClusterAPIKey: nil, + // No creation allowed. + crossClusterAPIKeyCreateRequests: nil, + }, + unexpectedSecrets: []types.NamespacedName{ + {Namespace: "ns1", Name: "es1-es-remote-api-keys"}, + {Namespace: "ns3", Name: "es3-es-remote-api-keys"}, + {Namespace: "ns4", Name: "es4-es-remote-api-keys"}, + }, + expectedKeystoreSecrets: []*corev1.Secret{ + { + // Keystore for es2 must not be updated with es1 key. + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"existing-api-key-to-esx":{"namespace":"foo","name":"bar","id":"apikey-to-esx"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "es2", + }, + Namespace: "ns2", + Name: "es2-es-remote-api-keys", + }, + Data: map[string][]byte{ + // This entry is going to be removed once ns2/es2 is reconciled. + "cluster.remote.existing-api-key-to-esx.credentials": []byte("encoded-key-for-existing-api-key"), + }, + }, + }, + }, + { + name: "Deleted remote cluster", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + unexpectedSecrets: []types.NamespacedName{ + { + Namespace: "ns1", + Name: remoteCASecretName("es1", types.NamespacedName{ + Namespace: "ns2", + Name: "es2", + }), + }, + { + Namespace: "ns2", + Name: remoteCASecretName("es2", types.NamespacedName{ + Namespace: "ns1", + Name: "es1", + }), + }, + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "CA content has been updated, remote ca must be reconciled", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + withDataCert(remoteCa("ns1", "es1", "ns2", "es2"), []byte("foo")), + withDataCert(remoteCa("ns2", "es2", "ns1", "es1"), []byte("bar")), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + // ns1/es1 has been deleted - all related secrets in other namespaces must be deleted + name: "Deleted cluster", + fields: fields{ + clusters: slices.Concat( + // ns2/es2 + newClusterBuilder("ns2", "es2", "7.0.0").withRemoteCluster("ns1", "es1").build(), + // ns3/es3 + newClusterBuilder("ns3", "es3", "7.0.0").withRemoteCluster("ns1", "es1").build(), + []client.Object{ + fakePublicCa("ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + fakePublicCa("ns3", "es3"), + remoteCa("ns3", "es3", "ns1", "es1"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + unexpectedSecrets: []types.NamespacedName{ + { + Namespace: "ns3", + Name: remoteCASecretName("es3", types.NamespacedName{ + Namespace: "ns1", + Name: "es1", + }), + }, + { + Namespace: "ns2", + Name: remoteCASecretName("es2", types.NamespacedName{ + Namespace: "ns1", + Name: "es1", + }), + }, + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "No enterprise license, remote ca are not created", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: false}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + unexpectedSecrets: []types.NamespacedName{ + { + Namespace: "ns1", + Name: remoteCASecretName("es1", types.NamespacedName{ + Namespace: "ns2", + Name: "es2", + }), + }, + { + Namespace: "ns2", + Name: remoteCASecretName("es2", types.NamespacedName{ + Namespace: "ns1", + Name: "es1", + }), + }, + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "No enterprise license, existing remote ca are left untouched", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: true}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: false}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + expectedCASecrets: []*corev1.Secret{ + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + want: reconcile.Result{}, + wantErr: false, + }, + { + name: "Association is not allowed, existing remote ca are removed", + fields: fields{ + clusters: slices.Concat( + newClusterBuilder("ns1", "es1", "7.0.0").withRemoteCluster("ns2", "es2").build(), + newClusterBuilder("ns2", "es2", "7.0.0").build(), + []client.Object{ + fakePublicCa("ns1", "es1"), + fakePublicCa("ns2", "es2"), + remoteCa("ns1", "es1", "ns2", "es2"), + remoteCa("ns2", "es2", "ns1", "es1"), + }, + ), + accessReviewer: &fakeAccessReviewer{allowed: false}, + licenseChecker: license.MockLicenseChecker{EnterpriseEnabled: true}, + }, + args: args{ + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "es1", + Namespace: "ns1", + }, + }, + }, + unexpectedSecrets: []types.NamespacedName{ + { + Namespace: "ns1", + Name: remoteCASecretName("es1", types.NamespacedName{Namespace: "ns2", Name: "es2"}), + }, + { + Namespace: "ns2", + Name: remoteCASecretName("es2", types.NamespacedName{Namespace: "ns1", Name: "es1"}), + }, + }, + want: reconcile.Result{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := watches.NewDynamicWatches() + fakeESClient := &fakeESClient{} + if tt.fakeESClient != nil { + fakeESClient = tt.fakeESClient + } + k8sClient := k8s.NewFakeClient(tt.fields.clusters...) + r := &ReconcileRemoteClusters{ + Client: k8sClient, + accessReviewer: tt.fields.accessReviewer, + watches: w, + licenseChecker: tt.fields.licenseChecker, + recorder: record.NewFakeRecorder(10), + esClientProvider: func(_ context.Context, _ k8s.Client, _ net.Dialer, _ esv1.Elasticsearch) (esclient.Client, error) { + return fakeESClient, nil + }, + keystoreProvider: keystore.NewProvider(k8sClient), + } + fakeCtx := context.Background() + got, err := r.Reconcile(fakeCtx, tt.args.request) + if (err != nil) != tt.wantErr { + t.Errorf("ReconcileRemoteCa.Reconcile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ReconcileRemoteCa.Reconcile() = %v, want %v", got, tt.want) + } + // Check that expected secrets are here + for _, expectedSecret := range tt.expectedCASecrets { + var actualSecret corev1.Secret + assert.NoError(t, r.Client.Get(context.Background(), types.NamespacedName{Namespace: expectedSecret.Namespace, Name: expectedSecret.Name}, &actualSecret)) + // Compare content + actualCa, ok := actualSecret.Data[certificates.CAFileName] + assert.True(t, ok) + assert.Equal(t, expectedSecret.Data[certificates.CAFileName], actualCa) + // Compare labels + assert.NotNil(t, actualSecret.Labels) + assert.Equal(t, expectedSecret.Labels, actualSecret.Labels) + } + // Check that unexpected secrets does not exist + for _, unexpectedSecret := range tt.unexpectedSecrets { + var actualSecret corev1.Secret + err := r.Client.Get(context.Background(), types.NamespacedName{Namespace: unexpectedSecret.Namespace, Name: unexpectedSecret.Name}, &actualSecret) + assert.True(t, apierrors.IsNotFound(err), "unexpected Secret %s/%s", unexpectedSecret.Namespace, unexpectedSecret.Name) + } + // Fake ES client assertions + assert.ElementsMatch(t, tt.wantEsAPICalls.getCrossClusterAPIKeys, fakeESClient.getCrossClusterAPIKeys, "unexpected calls to GetCrossClusterAPIKeys") + assert.ElementsMatch(t, tt.wantEsAPICalls.invalidateCrossClusterAPIKey, fakeESClient.invalidateCrossClusterAPIKey, "unexpected calls to InvalidateCrossClusterAPIKey") + assert.ElementsMatch( + t, + tt.wantEsAPICalls.crossClusterAPIKeyCreateRequests, + fakeESClient.crossClusterAPIKeyCreateRequests, + "unexpected calls to CreateCrossClusterAPIKey\n%s\n", cmp.Diff(tt.wantEsAPICalls.crossClusterAPIKeyCreateRequests, fakeESClient.crossClusterAPIKeyCreateRequests), + ) + assert.Equal( + t, + tt.wantEsAPICalls.updateCrossClusterAPIKey, + fakeESClient.updateCrossClusterAPIKey, + "unexpected calls to UpdateCrossClusterAPIKey\n%s\n", cmp.Diff(tt.wantEsAPICalls.updateCrossClusterAPIKey, fakeESClient.updateCrossClusterAPIKey), + ) + + // Keystore assertions + for _, expectedSecret := range tt.expectedKeystoreSecrets { + actualSecret := &corev1.Secret{} + if err := k8sClient.Get(fakeCtx, types.NamespacedName{ + Namespace: expectedSecret.Namespace, + Name: expectedSecret.Name, + }, actualSecret); err != nil { + t.Fatalf("error while retrieving keystore %s/%s: %v", expectedSecret.Namespace, expectedSecret.Name, err) + } + if diff := cmp.Diff(expectedSecret.Labels, actualSecret.Labels); len(diff) > 0 { + t.Errorf("unexpected labels on Secret %s/%s:\n%s\n", expectedSecret.Namespace, expectedSecret.Name, diff) + } + if diff := cmp.Diff(expectedSecret.Annotations, actualSecret.Annotations); len(diff) > 0 { + t.Errorf("unexpected annotations on Secret %s/%s:\n%s\n", expectedSecret.Namespace, expectedSecret.Name, diff) + } + if diff := cmp.Diff(expectedSecret.Data, actualSecret.Data); len(diff) > 0 { + t.Errorf("unexpected data in Secret %s/%s:\n%s\n", expectedSecret.Namespace, expectedSecret.Name, diff) + } + } + }) + } +} diff --git a/pkg/controller/remotecluster/fixtures.go b/pkg/controller/remotecluster/fixtures.go new file mode 100644 index 0000000000..7d361db704 --- /dev/null +++ b/pkg/controller/remotecluster/fixtures.go @@ -0,0 +1,197 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package remotecluster + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/certificates" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/certificates/transport" + esclient "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" +) + +// -- Fake ES client + +type fakeESClient struct { + esclient.Client + + // fake responses + getCrossClusterAPIKeys []string + + // recorded requests + invalidateCrossClusterAPIKey []string + crossClusterAPIKeyCreateRequests []esclient.CrossClusterAPIKeyCreateRequest + existingCrossClusterAPIKeys esclient.CrossClusterAPIKeyList + updateCrossClusterAPIKey map[string]esclient.CrossClusterAPIKeyUpdateRequest +} + +func (f *fakeESClient) CreateCrossClusterAPIKey(ctx context.Context, crossClusterAPIKeyCreateRequest esclient.CrossClusterAPIKeyCreateRequest) (esclient.CrossClusterAPIKeyCreateResponse, error) { + f.crossClusterAPIKeyCreateRequests = append(f.crossClusterAPIKeyCreateRequests, crossClusterAPIKeyCreateRequest) + return esclient.CrossClusterAPIKeyCreateResponse{ + ID: fmt.Sprintf("generated-id-from-fake-es-client-%s", crossClusterAPIKeyCreateRequest.Name), + Name: crossClusterAPIKeyCreateRequest.Name, + Encoded: fmt.Sprintf("generated-encoded-key-from-fake-es-client-for-%s", crossClusterAPIKeyCreateRequest.Name), + }, nil +} + +func (f *fakeESClient) GetCrossClusterAPIKeys(_ context.Context, name string) (esclient.CrossClusterAPIKeyList, error) { + f.getCrossClusterAPIKeys = append(f.getCrossClusterAPIKeys, name) + return f.existingCrossClusterAPIKeys, nil +} + +func (f *fakeESClient) InvalidateCrossClusterAPIKey(_ context.Context, name string) error { + f.invalidateCrossClusterAPIKey = append(f.invalidateCrossClusterAPIKey, name) + return nil +} + +func (f *fakeESClient) UpdateCrossClusterAPIKey(_ context.Context, name string, updateRequest esclient.CrossClusterAPIKeyUpdateRequest) (esclient.CrossClusterAPIKeyUpdateResponse, error) { + if f.updateCrossClusterAPIKey == nil { + f.updateCrossClusterAPIKey = make(map[string]esclient.CrossClusterAPIKeyUpdateRequest) + } + f.updateCrossClusterAPIKey[name] = updateRequest + return esclient.CrossClusterAPIKeyUpdateResponse{}, nil +} + +// -- Fake cluster builder + +type clusterBuilder struct { + name, namespace, version string + remoteClusters []esv1.RemoteCluster +} + +func newClusterBuilder(namespace, name, version string) *clusterBuilder { + return &clusterBuilder{ + name: name, + namespace: namespace, + version: version, + } +} + +func (cb *clusterBuilder) withRemoteCluster(namespace, name string) *clusterBuilder { + cb.remoteClusters = append(cb.remoteClusters, + esv1.RemoteCluster{ + Name: fmt.Sprintf("alias-from-%s-%s-to-%s-%s", cb.namespace, cb.name, namespace, name), + ElasticsearchRef: commonv1.LocalObjectSelector{ + Name: name, + Namespace: namespace, + }, + }) + return cb +} + +func (cb *clusterBuilder) withAPIKey(namespace, name string, apiKey *esv1.RemoteClusterAPIKey) *clusterBuilder { + cb.remoteClusters = append(cb.remoteClusters, + esv1.RemoteCluster{ + Name: fmt.Sprintf("generated-alias-from-%s-%s-to-%s-%s-with-api-key", cb.namespace, cb.name, namespace, name), + ElasticsearchRef: commonv1.LocalObjectSelector{ + Name: name, + Namespace: namespace, + }, + APIKey: apiKey, + }) + return cb +} + +func (cb *clusterBuilder) build() []client.Object { + remoteClusters := make([]esv1.RemoteCluster, len(cb.remoteClusters)) + for i, remoteCluster := range cb.remoteClusters { + remoteCluster := remoteCluster.DeepCopy() + remoteClusters[i] = *remoteCluster + } + return []client.Object{ + &esv1.Elasticsearch{ + ObjectMeta: v1.ObjectMeta{ + Namespace: cb.namespace, + Name: cb.name, + }, + Spec: esv1.ElasticsearchSpec{ + Version: cb.version, + RemoteClusters: remoteClusters, + }, + Status: esv1.ElasticsearchStatus{ + AvailableNodes: 3, + Version: cb.version, + }, + }, + &corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Namespace: cb.namespace, + Name: fmt.Sprintf("es-%s-%s-1", cb.namespace, cb.name), + Labels: map[string]string{ + label.ClusterNameLabelName: cb.name, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + } +} + +type fakeAccessReviewer struct { + allowed bool + err error +} + +func (f *fakeAccessReviewer) AccessAllowed(_ context.Context, _ string, _ string, _ runtime.Object) (bool, error) { + return f.allowed, f.err +} + +func fakePublicCa(namespace, name string) *corev1.Secret { + namespacedName := types.NamespacedName{ + Name: name, + Namespace: namespace, + } + transportPublicCertKey := transport.PublicCertsSecretRef(namespacedName) + return &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Namespace: transportPublicCertKey.Namespace, + Name: transportPublicCertKey.Name, + }, + Data: map[string][]byte{ + certificates.CAFileName: []byte(namespacedName.String()), + }, + } +} + +// remoteCa builds an expected remote Ca +func remoteCa(localNamespace, localName, remoteNamespace, remoteName string) *corev1.Secret { + remoteNamespacedName := types.NamespacedName{ + Name: remoteName, + Namespace: remoteNamespace, + } + return &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Namespace: localNamespace, + Name: remoteCASecretName(localName, remoteNamespacedName), + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-ca", + "elasticsearch.k8s.elastic.co/cluster-name": localName, + "elasticsearch.k8s.elastic.co/remote-cluster-name": remoteName, + "elasticsearch.k8s.elastic.co/remote-cluster-namespace": remoteNamespace, + }, + }, + Data: map[string][]byte{ + certificates.CAFileName: []byte(remoteNamespacedName.String()), + }, + } +} + +func withDataCert(caSecret *corev1.Secret, newCa []byte) *corev1.Secret { + caSecret.Data[certificates.CAFileName] = newCa + return caSecret +} diff --git a/pkg/controller/remotecluster/keystore/changes_tracker.go b/pkg/controller/remotecluster/keystore/changes_tracker.go new file mode 100644 index 0000000000..7975d64dc0 --- /dev/null +++ b/pkg/controller/remotecluster/keystore/changes_tracker.go @@ -0,0 +1,97 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package keystore + +import ( + "sync" +) + +// pendingChanges are changes which are expected to be eventually observed in the API keystore. +type pendingChanges struct { + // changes maps for each Alias (remote cluster name as defined in the client ES) the pending change. + changes map[string]pendingChange + mu sync.RWMutex +} + +type pendingChange struct { + remoteClusterName, remoteClusterNamespace, alias string + key key +} + +// key holds an expected key as generated by Elasticsearch, with its ID and its encoded value. +type key struct { + keyID, encodedValue string +} + +// AddKey registers a key to be eventually persisted in the underlying Secret. +func (pc *pendingChanges) AddKey(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue string) { + if pc == nil { + return + } + pc.mu.Lock() + defer pc.mu.Unlock() + pc.changes[alias] = pendingChange{ + remoteClusterName: remoteClusterName, + remoteClusterNamespace: remoteClusterNamespace, + alias: alias, + key: key{ + keyID: keyID, + encodedValue: encodedKeyValue, + }, + } +} + +// ForgetChangeFor must be called once we know that the key added with AddKey or removed with DeleteAlias is reflected in the underlying Secret. +func (pc *pendingChanges) ForgetChangeFor(alias string) { + if pc == nil { + return + } + pc.mu.Lock() + defer pc.mu.Unlock() + delete(pc.changes, alias) +} + +// DeleteAlias registers the deletion of the key for the provided alias. +func (pc *pendingChanges) DeleteAlias(alias string) { + if pc == nil { + return + } + pc.mu.Lock() + defer pc.mu.Unlock() + pc.changes[alias] = pendingChange{ + alias: alias, + key: key{}, // an empty key means that this alias must be deleted + } +} + +// Get returns all the pending changes. +func (pc *pendingChanges) Get() []pendingChange { + if pc == nil { + return nil + } + pc.mu.RLock() + defer pc.mu.RUnlock() + pendingChanges := make([]pendingChange, 0, len(pc.changes)) + for alias, change := range pc.changes { + pendingChange := pendingChange{ + remoteClusterName: change.remoteClusterName, + remoteClusterNamespace: change.remoteClusterNamespace, + alias: alias, + key: key{ + keyID: change.key.keyID, + encodedValue: change.key.encodedValue, + }, + } + pendingChanges = append(pendingChanges, pendingChange) + } + return pendingChanges +} + +func (k *key) IsEmpty() bool { + if k == nil { + return true + } + return k.encodedValue == "" && k.keyID == "" +} diff --git a/pkg/controller/remotecluster/keystore/keystore.go b/pkg/controller/remotecluster/keystore/keystore.go new file mode 100644 index 0000000000..a15527abc6 --- /dev/null +++ b/pkg/controller/remotecluster/keystore/keystore.go @@ -0,0 +1,300 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package keystore + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + + "github.com/go-logr/logr" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1" + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/labels" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/reconciler" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" +) + +const ( + aliasesAnnotationName = "elasticsearch.k8s.elastic.co/remote-cluster-api-keys" + + RemoteClusterAPIKeysType = "remote-cluster-api-keys" +) + +var ( + credentialsSecretSettingsRegEx = regexp.MustCompile(`^cluster\.remote\.([\w-]+)\.credentials$`) +) + +type APIKeyStore struct { + log logr.Logger + // aliases maps cluster aliased with the expected key ID + aliases map[string]AliasValue + // encodedKeys maps the remote cluster alias, as define in the client cluster, to the encoded cross-cluster API key. + encodedKeys map[string]string + // resourceVersion is the ResourceVersion as observed when the Secret has been loaded. + resourceVersion string + // uid is the UID of the Secret as observed when the Secret has been loaded. + uid types.UID + // pendingChanges are the pending changes, they are used to record changes until they are observed in the underlying Secret. + pendingChanges *pendingChanges +} + +type AliasValue struct { + // Namespace of the remote cluster. + Namespace string `json:"namespace"` + // Name of the remote cluster. + Name string `json:"name"` + // ID is the key ID. + ID string `json:"id"` +} + +func (aks *APIKeyStore) GetAliases() map[string]AliasValue { + if aks == nil { + return nil + } + return aks.aliases +} + +func (aks *APIKeyStore) KeyIDFor(alias string) string { + if aks == nil { + return "" + } + return aks.aliases[alias].ID +} + +func loadAPIKeyStore(ctx context.Context, log logr.Logger, c k8s.Client, owner *esv1.Elasticsearch, pendingChanges *pendingChanges) (*APIKeyStore, error) { + secretName := types.NamespacedName{ + Name: esv1.RemoteAPIKeysSecretName(owner.Name), + Namespace: owner.Namespace, + } + // Attempt to read the Secret + keyStoreSecret := &corev1.Secret{} + if err := c.Get(ctx, secretName, keyStoreSecret); err != nil { + if errors.IsNotFound(err) { + ulog.FromContext(ctx).V(1).Info("No APIKeyStore Secret found") + // Return an empty store + emptyKeystore := &APIKeyStore{log: log, pendingChanges: pendingChanges} + return emptyKeystore.withPendingChanges(), nil + } + } + + // Read the key aliased + aliases := make(map[string]AliasValue) + if aliasesAnnotation, ok := keyStoreSecret.Annotations[aliasesAnnotationName]; ok { + if err := json.Unmarshal([]byte(aliasesAnnotation), &aliases); err != nil { + return nil, err + } + } + + // Read the current encoded cross-cluster API keys. + encodedKeys := make(map[string]string) + for settingName, encodedAPIKey := range keyStoreSecret.Data { + strings := credentialsSecretSettingsRegEx.FindStringSubmatch(settingName) + if len(strings) != 2 { + ulog.FromContext(ctx).V(1).Info( + fmt.Sprintf("Unknown remote cluster credential setting: %s", settingName), + ) + continue + } + encodedKeys[strings[1]] = string(encodedAPIKey) + } + apiKeyStore := &APIKeyStore{ + log: log, + aliases: aliases, + encodedKeys: encodedKeys, + resourceVersion: keyStoreSecret.ResourceVersion, + uid: keyStoreSecret.UID, + pendingChanges: pendingChanges, + } + return apiKeyStore.withPendingChanges(), nil +} + +// withPendingChanges checks if the pending changes are reflected in the Secret. If it is the case these changes are removed from the expected changes. +// If not there are "virtually" added to the current keystore. +func (aks *APIKeyStore) withPendingChanges() *APIKeyStore { + pendingChanges := aks.pendingChanges.Get() + var pendingAdds, pendingDeletions int + for _, pendingChange := range pendingChanges { + if pendingChange.key.IsEmpty() { + if aks.KeyIDFor(pendingChange.alias) == "" { + aks.log.Info(fmt.Sprintf("Change for alias %s observed, key has been deleted in API keystore", pendingChange.alias)) + aks.pendingChanges.ForgetChangeFor(pendingChange.alias) + continue + } + // We are still expecting this deletion + pendingDeletions++ + aks.Delete(pendingChange.alias) + continue + } + // Check if the key is available in the underlying Secret + if keyIDInSecret := aks.KeyIDFor(pendingChange.alias); keyIDInSecret == pendingChange.key.keyID { + aks.log.Info(fmt.Sprintf("Change for alias %s observed, key %s saved in API keystore", pendingChange.alias, keyIDInSecret)) + // Forget this change + aks.pendingChanges.ForgetChangeFor(pendingChange.alias) + continue + } + // Change is not reflected in the Secret yet. + pendingAdds++ + aks.update(pendingChange.remoteClusterName, pendingChange.remoteClusterNamespace, pendingChange.alias, pendingChange.key.keyID, pendingChange.key.encodedValue) + } + + if pendingAdds > 0 || pendingDeletions > 0 { + aks.log.Info("Pending changes in API keystore", "add", pendingAdds, "deletion", pendingDeletions) + } + return aks +} + +func (aks *APIKeyStore) Update(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue string) *APIKeyStore { + // Save the change in memory + aks.pendingChanges.AddKey(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue) + // Load the change in this instance of the store + aks.update(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue) + return aks +} + +func (aks *APIKeyStore) update(remoteClusterName, remoteClusterNamespace, alias, keyID, encodedKeyValue string) { + if aks.aliases == nil { + aks.aliases = make(map[string]AliasValue) + } + aks.aliases[alias] = AliasValue{ + Namespace: remoteClusterNamespace, + Name: remoteClusterName, + ID: keyID, + } + if aks.encodedKeys == nil { + aks.encodedKeys = make(map[string]string) + } + aks.encodedKeys[alias] = encodedKeyValue +} + +func (aks *APIKeyStore) Aliases() []string { + if aks == nil { + return nil + } + aliases := make([]string, len(aks.aliases)) + i := 0 + for alias := range aks.aliases { + aliases[i] = alias + i++ + } + return aliases +} + +func (aks *APIKeyStore) Delete(alias string) *APIKeyStore { + // Save the change in memory + aks.pendingChanges.DeleteAlias(alias) + // Load the change in this instance of the store + aks.delete(alias) + return aks +} + +func (aks *APIKeyStore) delete(alias string) *APIKeyStore { + delete(aks.aliases, alias) + delete(aks.encodedKeys, alias) + return aks +} + +const ( + credentialsKeyFormat = "cluster.remote.%s.credentials" +) + +// Save synchronizes the in memory content of the API keystore into the Secret. +func (aks *APIKeyStore) Save(ctx context.Context, c k8s.Client, owner *esv1.Elasticsearch) *reconciler.Results { + secretName := types.NamespacedName{ + Name: esv1.RemoteAPIKeysSecretName(owner.Name), + Namespace: owner.Namespace, + } + if aks.IsEmpty() { + return aks.deleteSecret(ctx, c, secretName) + } + + results := &reconciler.Results{} + aliases, err := json.Marshal(aks.aliases) + if err != nil { + return results.WithError(err) + } + data := make(map[string][]byte, len(aks.encodedKeys)) + for k, v := range aks.encodedKeys { + data[fmt.Sprintf(credentialsKeyFormat, k)] = []byte(v) + } + expectedLabels := labels.AddCredentialsLabel(label.NewLabels(k8s.ExtractNamespacedName(owner))) + expectedLabels[commonv1.TypeLabelName] = RemoteClusterAPIKeysType + expected := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName.Name, + Namespace: secretName.Namespace, + Annotations: map[string]string{ + aliasesAnnotationName: string(aliases), + }, + Labels: expectedLabels, + }, + Data: data, + } + if _, err := reconciler.ReconcileSecret(ctx, c, expected, owner); err != nil { + if errors.IsConflict(err) { + return results.WithResult(reconcile.Result{Requeue: true}) + } + return results.WithError(err) + } + return nil +} + +func (aks *APIKeyStore) deleteSecret(ctx context.Context, c k8s.Client, secretName types.NamespacedName) *reconciler.Results { + // Delete the Secret used to load the current state. + deleteOptions := make([]client.DeleteOption, 0, 2) + if aks.uid != "" { + deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &aks.uid}}) + } + if aks.resourceVersion != "" { + deleteOptions = append(deleteOptions, &client.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &aks.resourceVersion}}) + } + results := &reconciler.Results{} + if err := c.Delete(ctx, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName.Name, Namespace: secretName.Namespace}}, + deleteOptions..., + ); err != nil { + if errors.IsNotFound(err) { + return nil + } + if errors.IsConflict(err) { + return results.WithResult(reconcile.Result{Requeue: true}) + } + return results.WithError(err) + } + return nil +} + +func (aks *APIKeyStore) IsEmpty() bool { + if aks == nil { + return true + } + return len(aks.aliases) == 0 +} + +// ForCluster returns +func (aks *APIKeyStore) ForCluster(namespace string, name string) sets.Set[string] { + aliases := sets.New[string]() + if aks == nil { + return aliases + } + for alias, c := range aks.aliases { + if c.Name == name && c.Namespace == namespace { + aliases.Insert(alias) + } + } + return aliases +} diff --git a/pkg/controller/remotecluster/keystore/keystore_test.go b/pkg/controller/remotecluster/keystore/keystore_test.go new file mode 100644 index 0000000000..b980eb7a06 --- /dev/null +++ b/pkg/controller/remotecluster/keystore/keystore_test.go @@ -0,0 +1,287 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package keystore + +import ( + "context" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log" +) + +const ( + testNamespace = "ns1" +) + +var ( + es = &esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myes", + Namespace: testNamespace, + UID: uuid.NewUUID(), + Generation: 42, + }, + Spec: esv1.ElasticsearchSpec{}, + } +) + +func TestLoadAPIKeyStore(t *testing.T) { + type args struct { + c k8s.Client + owner *esv1.Elasticsearch + pendingChanges *pendingChanges + } + tests := []struct { + name string + args args + want *APIKeyStore + wantErr bool + }{ + { + name: "Happy path", + args: args{ + c: k8s.NewFakeClient( + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{ "rc2" : { "namespace" : "ns2", "name" : "es2", "id": "SecretKeyID2" }, "rc1" : { "namespace" : "ns1", "name" : "es1", "id": "SecretKeyID1" } }`, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc1.credentials": []byte("SecretKeyValue1"), + "cluster.remote.rc2.credentials": []byte("SecretKeyValue2"), + }, + }, + ), + owner: es, + }, + want: &APIKeyStore{ + aliases: map[string]AliasValue{ + "rc1": {ID: "SecretKeyID1", Namespace: "ns1", Name: "es1"}, + "rc2": {ID: "SecretKeyID2", Namespace: "ns2", Name: "es2"}, + }, + encodedKeys: map[string]string{ + "rc1": "SecretKeyValue1", + "rc2": "SecretKeyValue2", + }, + }, + }, + { + name: "Secret does not exist", + args: args{ + c: k8s.NewFakeClient(), + owner: es, + }, + want: &APIKeyStore{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + got, err := loadAPIKeyStore(ctx, ulog.Log, tt.args.c, tt.args.owner, tt.args.pendingChanges) + if (err != nil) != tt.wantErr { + t.Errorf("loadAPIKeyStore() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got.encodedKeys, tt.want.encodedKeys) { + t.Errorf("loadAPIKeyStore().keys = %v, want.keys %v", got.encodedKeys, tt.want.encodedKeys) + } + if !reflect.DeepEqual(got.aliases, tt.want.aliases) { + t.Errorf("loadAPIKeyStore().aliases = %v, want.aliases %v", got.aliases, tt.want.aliases) + } + }) + } +} + +func TestAPIKeyStore_Save(t *testing.T) { + type args struct { + c k8s.Client + } + tests := []struct { + name string + receiver *APIKeyStore + args args + wantErr bool + want *corev1.Secret + }{ + { + name: "Create a new store", + receiver: (&APIKeyStore{}). + Update("ns1", "es1", "rc1", "keyid1", "encodedValue1"). + Update("ns1", "es1", "rc2", "keyid2", "encodedValue2"), + args: args{c: k8s.NewFakeClient()}, + want: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + ResourceVersion: "1", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"rc1":{"namespace":"es1","name":"ns1","id":"keyid1"},"rc2":{"namespace":"es1","name":"ns1","id":"keyid2"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "myes", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "elasticsearch.k8s.elastic.co/v1", + Kind: "Elasticsearch", + Name: "myes", + UID: es.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc1.credentials": []byte("encodedValue1"), + "cluster.remote.rc2.credentials": []byte("encodedValue2"), + }, + }, + }, + { + name: "Delete the store", + receiver: (&APIKeyStore{}). + Update("ns1", "es1", "rc1", "keyid1", "encodedValue1"). + Update("ns2", "es2", "rc2", "keyid2", "encodedValue2"). + Delete("rc1").Delete("rc2"), + args: args{c: k8s.NewFakeClient(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + ResourceVersion: "1", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"rc1":"keyid1","rc2":"keyid2"}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "elasticsearch", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "myes", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "elasticsearch.k8s.elastic.co/v1", + Kind: "Elasticsearch", + Name: "myes", + UID: es.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc1.credentials": []byte("encodedValue1"), + "cluster.remote.rc2.credentials": []byte("encodedValue2"), + }, + })}, + }, + { + name: "AddKey new keys, remove another", + receiver: (&APIKeyStore{}). + Update("ns1", "es1", "rc1", "keyid1", "encodedValue1"). + Update("ns2", "es2", "rc2", "keyid2", "encodedValue2"). + Delete("rc1"). + Update("ns3", "es3", "rc3_1", "keyid3_1", "encodedValue31"). + Update("ns3", "es3", "rc3_2", "keyid3_2", "encodedValue32"), + args: args{c: k8s.NewFakeClient(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + ResourceVersion: "1", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"rc2":{"namespace":"es2","name":"ns2","id":"keyid2"},"rc1":{"namespace":"es1","name":"ns1","id":"keyid1"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "myes", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "elasticsearch.k8s.elastic.co/v1", + Kind: "Elasticsearch", + Name: "myes", + UID: es.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc1.credentials": []byte("encodedValue1"), + "unexpected_key": []byte("foo"), + "cluster.remote.rc2.credentials": []byte("encodedValue2"), + }, + })}, + want: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "myes-es-remote-api-keys", + ResourceVersion: "2", + Annotations: map[string]string{ + "elasticsearch.k8s.elastic.co/remote-cluster-api-keys": `{"rc2":{"namespace":"es2","name":"ns2","id":"keyid2"},"rc3_1":{"namespace":"es3","name":"ns3","id":"keyid3_1"},"rc3_2":{"namespace":"es3","name":"ns3","id":"keyid3_2"}}`, + }, + Labels: map[string]string{ + "common.k8s.elastic.co/type": "remote-cluster-api-keys", + "eck.k8s.elastic.co/credentials": "true", + "elasticsearch.k8s.elastic.co/cluster-name": "myes", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "elasticsearch.k8s.elastic.co/v1", + Kind: "Elasticsearch", + Name: "myes", + UID: es.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Data: map[string][]byte{ + "cluster.remote.rc2.credentials": []byte("encodedValue2"), + "cluster.remote.rc3_1.credentials": []byte("encodedValue31"), + "cluster.remote.rc3_2.credentials": []byte("encodedValue32"), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + owner := es.DeepCopy() + if err := tt.receiver.Save(ctx, tt.args.c, owner); (err != nil) != tt.wantErr { + t.Errorf("APIKeyStore.Save() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.want != nil { + // get the Secret + apiKeysSecret := corev1.Secret{} + assert.NoError(t, tt.args.c.Get(ctx, types.NamespacedName{Name: "myes-es-remote-api-keys", Namespace: testNamespace}, &apiKeysSecret)) + if diff := cmp.Diff(*tt.want, apiKeysSecret); len(diff) > 0 { + t.Errorf("%s", diff) + } + } else { + // ensure the Secret does not exist + err := tt.args.c.Get(ctx, types.NamespacedName{Name: "myes-es-remote-api-keys", Namespace: testNamespace}, &corev1.Secret{}) + assert.Truef(t, errors.IsNotFound(err), "expected a 404 error") + } + }) + } +} diff --git a/pkg/controller/remotecluster/keystore/provider.go b/pkg/controller/remotecluster/keystore/provider.go new file mode 100644 index 0000000000..8fd9cdb177 --- /dev/null +++ b/pkg/controller/remotecluster/keystore/provider.go @@ -0,0 +1,87 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package keystore + +import ( + "context" + "sync" + + "github.com/go-logr/logr" + + "k8s.io/apimachinery/pkg/types" + + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s" +) + +type pendingChangesPerCluster struct { + pendingChangesPerCluster map[types.NamespacedName]*pendingChanges + mu sync.RWMutex +} + +func NewProvider(c k8s.Client) *Provider { + return &Provider{ + c: c, + pendingChangesPerCluster: pendingChangesPerCluster{ + pendingChangesPerCluster: make(map[types.NamespacedName]*pendingChanges), + }, + } +} + +type Provider struct { + c k8s.Client + pendingChangesPerCluster pendingChangesPerCluster +} + +func (p *Provider) ForgetCluster(name types.NamespacedName) { + if p == nil { + return + } + p.pendingChangesPerCluster.mu.Lock() + defer p.pendingChangesPerCluster.mu.Unlock() + delete(p.pendingChangesPerCluster.pendingChangesPerCluster, name) +} + +func (p *Provider) ForCluster(ctx context.Context, log logr.Logger, owner *esv1.Elasticsearch) (*APIKeyStore, error) { + if p == nil { + return nil, nil + } + name := types.NamespacedName{ + Namespace: owner.Namespace, + Name: owner.Name, + } + pendingChanges := p.forCluster(name) + if pendingChanges != nil { + return loadAPIKeyStore(ctx, log, p.c, owner, pendingChanges) + } + return loadAPIKeyStore(ctx, log, p.c, owner, p.newForCluster(name)) +} + +func (p *Provider) forCluster(name types.NamespacedName) *pendingChanges { + if p == nil { + return nil + } + p.pendingChangesPerCluster.mu.RLock() + defer p.pendingChangesPerCluster.mu.RUnlock() + return p.pendingChangesPerCluster.pendingChangesPerCluster[name] +} + +func (p *Provider) newForCluster(name types.NamespacedName) *pendingChanges { + if p == nil { + return nil + } + p.pendingChangesPerCluster.mu.Lock() + defer p.pendingChangesPerCluster.mu.Unlock() + // Check if another goroutine did not create the pending changes + currentPendingChanges := p.pendingChangesPerCluster.pendingChangesPerCluster[name] + if currentPendingChanges != nil { + return currentPendingChanges + } + newPendingChanges := &pendingChanges{ + changes: make(map[string]pendingChange), + } + p.pendingChangesPerCluster.pendingChangesPerCluster[name] = newPendingChanges + return newPendingChanges +} diff --git a/pkg/controller/remoteca/labels.go b/pkg/controller/remotecluster/labels.go similarity index 98% rename from pkg/controller/remoteca/labels.go rename to pkg/controller/remotecluster/labels.go index edb2789c12..57bb5d3323 100644 --- a/pkg/controller/remoteca/labels.go +++ b/pkg/controller/remotecluster/labels.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "fmt" diff --git a/pkg/controller/remoteca/rbac.go b/pkg/controller/remotecluster/rbac.go similarity index 98% rename from pkg/controller/remoteca/rbac.go rename to pkg/controller/remotecluster/rbac.go index 991b129a8f..d5d92e9a9e 100644 --- a/pkg/controller/remoteca/rbac.go +++ b/pkg/controller/remotecluster/rbac.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "context" diff --git a/pkg/controller/remoteca/secret.go b/pkg/controller/remotecluster/secret.go similarity index 98% rename from pkg/controller/remoteca/secret.go rename to pkg/controller/remotecluster/secret.go index 6c09e518b0..605d2d1bc2 100644 --- a/pkg/controller/remoteca/secret.go +++ b/pkg/controller/remotecluster/secret.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "context" @@ -29,7 +29,7 @@ import ( // * Copy the CA of the remote cluster to the local one. func createOrUpdateCertificateAuthorities( ctx context.Context, - r *ReconcileRemoteCa, + r *ReconcileRemoteClusters, local, remote *esv1.Elasticsearch, ) *reconciler.Results { span, _ := apm.StartSpan(ctx, "create_or_update_remote_ca", tracing.SpanTypeApp) @@ -79,7 +79,7 @@ func createOrUpdateCertificateAuthorities( // copyCertificateAuthority creates a copy of the CA from a source cluster to a target cluster func copyCertificateAuthority( ctx context.Context, - r *ReconcileRemoteCa, + r *ReconcileRemoteClusters, source, target *esv1.Elasticsearch, ) error { sourceKey := k8s.ExtractNamespacedName(source) @@ -110,7 +110,7 @@ func copyCertificateAuthority( // remote cluster must be deleted from the local one. func deleteCertificateAuthorities( ctx context.Context, - r *ReconcileRemoteCa, + r *ReconcileRemoteClusters, local, remote types.NamespacedName, ) error { span, ctx := apm.StartSpan(ctx, "delete_certificate_authorities", tracing.SpanTypeApp) diff --git a/pkg/controller/remoteca/watches.go b/pkg/controller/remotecluster/watches.go similarity index 62% rename from pkg/controller/remoteca/watches.go rename to pkg/controller/remotecluster/watches.go index db5f852473..350b68d329 100644 --- a/pkg/controller/remoteca/watches.go +++ b/pkg/controller/remotecluster/watches.go @@ -2,12 +2,16 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package remoteca +package remotecluster import ( "context" "fmt" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/remotecluster/keystore" + + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -26,13 +30,34 @@ import ( ) // AddWatches set watches on objects needed to manage the association between a local and a remote cluster. -func addWatches(mgr manager.Manager, c controller.Controller, r *ReconcileRemoteCa) error { +func addWatches(mgr manager.Manager, c controller.Controller, r *ReconcileRemoteClusters) error { // Watch for changes to RemoteCluster if err := c.Watch(source.Kind(mgr.GetCache(), &esv1.Elasticsearch{}, &handler.TypedEnqueueRequestForObject[*esv1.Elasticsearch]{})); err != nil { return err } - // Watch Secrets that contain remote certificate authorities managed by this controller + // Emit changes to remote clusters to update API keys. + if err := c.Watch( + source.Kind( + mgr.GetCache(), + &esv1.Elasticsearch{}, + handler.TypedEnqueueRequestsFromMapFunc[*esv1.Elasticsearch, reconcile.Request]( + func(ctx context.Context, elasticsearch *esv1.Elasticsearch) []reconcile.Request { + requests := make([]reconcile.Request, 0, len(elasticsearch.Spec.RemoteClusters)) + for _, remoteCluster := range elasticsearch.Spec.RemoteClusters { + requests = append(requests, reconcile.Request{NamespacedName: remoteCluster.ElasticsearchRef.WithDefaultNamespace(elasticsearch.Namespace).NamespacedName()}) + } + return requests + }, + ), + ), + ); err != nil { + return err + } + + // Watch Secrets that contain: + // * Remote certificate authorities managed by this controller. + // * API keys if err := c.Watch( source.Kind(mgr.GetCache(), &v1.Secret{}, handler.TypedEnqueueRequestsFromMapFunc[*v1.Secret, reconcile.Request](newRequestsFromMatchedLabels()), @@ -58,21 +83,36 @@ func addWatches(mgr manager.Manager, c controller.Controller, r *ReconcileRemote // newRequestsFromMatchedLabels creates a watch handler function that creates reconcile requests based on the // labels set on a Secret which contains the remote CA. func newRequestsFromMatchedLabels() handler.TypedMapFunc[*v1.Secret, reconcile.Request] { - return handler.TypedMapFunc[*v1.Secret, reconcile.Request](func(ctx context.Context, obj *v1.Secret) []reconcile.Request { + return func(ctx context.Context, obj *v1.Secret) []reconcile.Request { labels := obj.GetLabels() - if !maps.ContainsKeys(labels, RemoteClusterNameLabelName, RemoteClusterNamespaceLabelName, commonv1.TypeLabelName) { - return nil + if maps.ContainsKeys(labels, RemoteClusterNameLabelName, RemoteClusterNamespaceLabelName, commonv1.TypeLabelName) { + // Remote cluster CA + if labels[commonv1.TypeLabelName] != remoteca.TypeLabelValue { + return nil + } + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Namespace: labels[RemoteClusterNamespaceLabelName], + Name: labels[RemoteClusterNameLabelName]}, + }, + } } - if labels[commonv1.TypeLabelName] != remoteca.TypeLabelValue { - return nil - } - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Namespace: labels[RemoteClusterNamespaceLabelName], - Name: labels[RemoteClusterNameLabelName]}, - }, + + if maps.ContainsKeys(labels, label.ClusterNameLabelName, commonv1.TypeLabelName) { + if labels[commonv1.TypeLabelName] != keystore.RemoteClusterAPIKeysType { + return nil + } + // Remote cluster API keys Secret event. + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Namespace: obj.Namespace, + Name: labels[label.ClusterNameLabelName]}, + }, + } } - }) + + return nil + } } func watchName(local types.NamespacedName, remote types.NamespacedName) string { @@ -89,7 +129,7 @@ func watchName(local types.NamespacedName, remote types.NamespacedName) string { // The local CA is watched to update the trusted certificates in the remote clusters. // The remote CAs are watched to update the trusted certificates of the local cluster. func addCertificatesAuthorityWatches( - reconcileClusterAssociation *ReconcileRemoteCa, + reconcileClusterAssociation *ReconcileRemoteClusters, local, remote types.NamespacedName) error { // Watch the CA secret of Elasticsearch clusters which are involved in a association. err := reconcileClusterAssociation.watches.Secrets.AddHandler(watches.NamedWatch[*corev1.Secret]{ diff --git a/pkg/telemetry/fixtures.go b/pkg/telemetry/fixtures.go index 0b817787c3..b5087f719e 100644 --- a/pkg/telemetry/fixtures.go +++ b/pkg/telemetry/fixtures.go @@ -21,7 +21,8 @@ type ElasticsearchTemplateData struct { StackMonitoringLogsCount int32 StackMonitoringMetricsCount int32 PodCount int32 - + RemoteClustersCount int32 + RemoteClustersAPIKeysCount int32 *NodeLabelsTemplateData } @@ -79,6 +80,8 @@ const expectedTelemetryTemplate = `eck: {{- end }} helm_resource_count: 0 pod_count: {{ .ElasticsearchTemplateData.PodCount }} + remote_clusters_api_keys_count: {{ .ElasticsearchTemplateData.RemoteClustersAPIKeysCount }} + remote_clusters_count: {{ .ElasticsearchTemplateData.RemoteClustersCount }} resource_count: {{ .ElasticsearchTemplateData.ResourceCount }} stack_monitoring_logs_count: {{ .ElasticsearchTemplateData.StackMonitoringLogsCount }} stack_monitoring_metrics_count: {{ .ElasticsearchTemplateData.StackMonitoringMetricsCount }} diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index 70f726518d..660c1e91f2 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -232,6 +232,8 @@ func esStats(k8sClient k8s.Client, managedNamespaces []string) (string, interfac AutoscaledResourceCount int32 `json:"autoscaled_resource_count"` StackMonitoringLogsCount int32 `json:"stack_monitoring_logs_count"` StackMonitoringMetricsCount int32 `json:"stack_monitoring_metrics_count"` + RemoteClustersCount int32 `json:"remote_clusters_count"` + RemoteClustersAPIKeysCount int32 `json:"remote_clusters_api_keys_count"` DownwardNodeLabels *downwardNodeLabelsStats `json:"downward_node_labels,omitempty"` }{} distinctNodeLabels := set.Make() @@ -247,6 +249,10 @@ func esStats(k8sClient k8s.Client, managedNamespaces []string) (string, interfac stats.ResourceCount++ stats.PodCount += es.Status.AvailableNodes + rcWithoutAPIKeys, rcWithAPIKeys := es.RemoteClustersCount() + stats.RemoteClustersCount += rcWithoutAPIKeys + stats.RemoteClustersAPIKeysCount += rcWithAPIKeys + if isManagedByHelm(es.Labels) { stats.HelmManagedResourceCount++ } diff --git a/pkg/telemetry/telemetry_test.go b/pkg/telemetry/telemetry_test.go index 7c54ab149b..08a29651fa 100644 --- a/pkg/telemetry/telemetry_test.go +++ b/pkg/telemetry/telemetry_test.go @@ -471,6 +471,8 @@ func TestNewReporter(t *testing.T) { autoscaled_resource_count: 2 helm_resource_count: 1 pod_count: 10 + remote_clusters_api_keys_count: 0 + remote_clusters_count: 0 resource_count: 4 stack_monitoring_logs_count: 1 stack_monitoring_metrics_count: 1 @@ -597,6 +599,8 @@ func TestReporter_report(t *testing.T) { PodCount: 9, ResourceCount: 3, StackMonitoringMetricsCount: 2, + RemoteClustersCount: 0, + RemoteClustersAPIKeysCount: 0, }, }, }, @@ -631,9 +635,11 @@ func TestReporter_report(t *testing.T) { }, wantData: TemplateData{ ElasticsearchTemplateData{ - PodCount: 10, - ResourceCount: 2, - StackMonitoringLogsCount: 1, + PodCount: 10, + ResourceCount: 2, + StackMonitoringLogsCount: 1, + RemoteClustersCount: 0, + RemoteClustersAPIKeysCount: 0, }, }, }, { @@ -665,8 +671,10 @@ func TestReporter_report(t *testing.T) { }, wantData: TemplateData{ ElasticsearchTemplateData{ - PodCount: 4, - ResourceCount: 2, + PodCount: 4, + ResourceCount: 2, + RemoteClustersCount: 0, + RemoteClustersAPIKeysCount: 0, NodeLabelsTemplateData: &NodeLabelsTemplateData{ ResourceWithNodeLabelsCount: 1, DistinctNodeLabelsCount: 1, @@ -705,8 +713,10 @@ func TestReporter_report(t *testing.T) { }, wantData: TemplateData{ ElasticsearchTemplateData{ - PodCount: 4, - ResourceCount: 2, + PodCount: 4, + ResourceCount: 2, + RemoteClustersCount: 0, + RemoteClustersAPIKeysCount: 0, NodeLabelsTemplateData: &NodeLabelsTemplateData{ ResourceWithNodeLabelsCount: 2, DistinctNodeLabelsCount: 3, // ns/label1,ns/label2 and ns/label3 @@ -714,6 +724,67 @@ func TestReporter_report(t *testing.T) { }, }, }, + { + name: "With remote clusters", + fields: fields{ + objects: []client.Object{ + &esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: "es1", + }, + Spec: esv1.ElasticsearchSpec{ + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "rc1", + APIKey: nil, + }, + { + Name: "rc2", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + }, + }, + Status: esv1.ElasticsearchStatus{ + AvailableNodes: 2, + }, + }, + &esv1.Elasticsearch{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: "es2", + }, + Spec: esv1.ElasticsearchSpec{ + RemoteClusters: []esv1.RemoteCluster{ + { + Name: "rc1", + APIKey: nil, + }, + { + Name: "rc2", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + { + Name: "rc3", + APIKey: &esv1.RemoteClusterAPIKey{}, + }, + }, + }, + Status: esv1.ElasticsearchStatus{ + AvailableNodes: 8, + }, + }, + }, + }, + wantData: TemplateData{ + ElasticsearchTemplateData{ + PodCount: 10, + ResourceCount: 2, + RemoteClustersCount: 1 + 1, + RemoteClustersAPIKeysCount: 1 + 2, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/utils/k8s/k8sutils.go b/pkg/utils/k8s/k8sutils.go index a68c16d769..a676052b20 100644 --- a/pkg/utils/k8s/k8sutils.go +++ b/pkg/utils/k8s/k8sutils.go @@ -226,6 +226,25 @@ func DeleteSecretIfExists(ctx context.Context, c Client, key types.NamespacedNam return err } +// DeleteResourceIfExists deletes the provided resource if exists. +func DeleteResourceIfExists(ctx context.Context, c Client, obj client.Object) error { + key := types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + if err := c.Get(ctx, key, obj); err != nil { + if apierrors.IsNotFound(err) { + // Resource does not exist. + return nil + } + return err + } + if err := c.Delete(ctx, obj); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + // PodsMatchingLabels returns Pods from the given namespace matching the given labels. func PodsMatchingLabels(c Client, namespace string, labels map[string]string) ([]corev1.Pod, error) { var pods corev1.PodList diff --git a/test/e2e/es/remote_cluster_test.go b/test/e2e/es/remote_cluster_test.go index ecbaf11237..240b173a61 100644 --- a/test/e2e/es/remote_cluster_test.go +++ b/test/e2e/es/remote_cluster_test.go @@ -12,10 +12,13 @@ import ( "fmt" "net/http" "os" + "strings" "testing" "github.com/stretchr/testify/require" + esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/client" "github.com/elastic/cloud-on-k8s/v2/test/e2e/test" "github.com/elastic/cloud-on-k8s/v2/test/e2e/test/elasticsearch" @@ -28,7 +31,7 @@ const ( // TestRemoteCluster tests the local K8S remote cluster feature. // 1. In a first Elasticsearch cluster some data are indexed in the "data-integrity-check" index. // 2. A second cluster is created with the first cluster declared as a remote cluster. -// 3. Finally an index follower is created in the second cluster to follow the "data-integrity-check" index from the first one. +// 3. Finally, an index follower is created in the second cluster to follow the "data-integrity-check" index from the first one. func TestRemoteCluster(t *testing.T) { // only execute this test if we have a test license to work with if test.Ctx().TestLicense == "" { @@ -85,6 +88,10 @@ func TestRemoteCluster(t *testing.T) { require.NoError(t, elasticsearch.NewDataIntegrityCheck(k, es1Builder).WithSoftDeletesEnabled(true).Init()) }, }, + test.Step{ + Name: "Check that remote cluster is connected using transport protocol", + Test: test.Eventually(checkRemoteClusterSeed(k, es2Builder, es1Builder, 9300)), + }, test.Step{ Name: "Create a follower index on the second cluster", Test: func(t *testing.T) { @@ -114,3 +121,148 @@ func TestRemoteCluster(t *testing.T) { test.Sequence(before, stepsFn, es1Builder, es2Builder).RunSequential(t) } + +// TestRemoteCluster tests the K8S remote cluster feature using API Keys. +// This test is similar to TestRemoteCluster: +// 1. In a first Elasticsearch cluster some data are indexed in the "data-integrity-check" index. +// 2. A second cluster is created with the first cluster declared as a remote cluster. +// 3. Finally, an index follower is created in the second cluster to follow the "data-integrity-check" index from the first one. +func TestRemoteClusterWithAPIKeys(t *testing.T) { + // only execute this test if we have a test license to work with + if test.Ctx().TestLicense == "" { + t.SkipNow() + } + + // only execute the test if Elasticsearch supports API keys + if version.MustParse(test.Ctx().ElasticStackVersion).LT(esv1.RemoteClusterAPIKeysMinVersion) { + t.Skipf("%s does not support remote cluster API keys", esv1.RemoteClusterAPIKeysMinVersion) + } + + name := "test-remote-cluster-api-keys" + ns1 := test.Ctx().ManagedNamespace(0) + es1Builder := elasticsearch.NewBuilder(name). + WithNamespace(ns1). + WithESMasterDataNodes(1, elasticsearch.DefaultResources). + WithRemoteClusterServer(). + WithRestrictedSecurityContext() + es1LicenseTestContext := elasticsearch.NewLicenseTestContext(test.NewK8sClientOrFatal(), es1Builder.Elasticsearch) + + ns2 := test.Ctx().ManagedNamespace(1) + es2Builder := elasticsearch.NewBuilder(name). + WithNamespace(ns2). + WithESMasterDataNodes(1, elasticsearch.DefaultResources). + WithRestrictedSecurityContext(). + WithRemoteClusterAPIKey(es1Builder, &esv1.RemoteClusterAPIKey{ + Access: esv1.RemoteClusterAccess{ + Search: &esv1.Search{ + Names: []string{elasticsearch.DataIntegrityIndex}, + }, + Replication: &esv1.Replication{ + Names: []string{elasticsearch.DataIntegrityIndex}, + }, + }, + }) + es2LicenseTestContext := elasticsearch.NewLicenseTestContext(test.NewK8sClientOrFatal(), es2Builder.Elasticsearch) + licenseBytes, err := os.ReadFile(test.Ctx().TestLicense) + require.NoError(t, err) + trialSecretName := "eck-license" + + before := func(k *test.K8sClient) test.StepList { + // Deploy a Trial license + return test.StepList{ + es1LicenseTestContext.DeleteAllEnterpriseLicenseSecrets(), + es1LicenseTestContext.CreateEnterpriseLicenseSecret(trialSecretName, licenseBytes), + } + } + + followerIndex := "data-integrity-check-follower" + remoteIndexName := fmt.Sprintf("%s:%s", es1Builder.Name(), elasticsearch.DataIntegrityIndex) + stepsFn := func(k *test.K8sClient) test.StepList { + return test.StepList{ + // Init license test context + es1LicenseTestContext.Init(), + es2LicenseTestContext.Init(), + // Check that the first cluster is using a Platinum license + es1LicenseTestContext.CheckElasticsearchLicense( + client.ElasticsearchLicenseTypeEnterprise, + ), + // Check that the second cluster is using a Platinum license + es1LicenseTestContext.CheckElasticsearchLicense( + client.ElasticsearchLicenseTypeEnterprise, + ), + test.Step{ + Name: "Add some data to the first cluster", + Test: func(t *testing.T) { + require.NoError(t, elasticsearch.NewDataIntegrityCheck(k, es1Builder).Init()) + }, + }, + test.Step{ + Name: "Check that remote cluster is connected to remote cluster server", + Test: test.Eventually(checkRemoteClusterSeed(k, es2Builder, es1Builder, 9443)), + }, + test.Step{ + Name: fmt.Sprintf("Check we can read remote index %s from the client cluster", remoteIndexName), + Test: test.Eventually(func() error { + return elasticsearch.NewDataIntegrityCheck(k, es2Builder).ForIndex(remoteIndexName).Verify() + }), + }, + test.Step{ + Name: "Create a follower index in the client cluster", + Test: func(t *testing.T) { + esClient, err := elasticsearch.NewElasticsearchClient(es2Builder.Elasticsearch, k) + require.NoError(t, err) + // create the index with controlled settings + followerCreation, err := http.NewRequest( + http.MethodPut, + fmt.Sprintf("/%s/_ccr/follow", followerIndex), + bytes.NewBufferString(fmt.Sprintf(followerSetting, es1Builder.Ref().Name, elasticsearch.DataIntegrityIndex)), + ) + require.NoError(t, err) + resp, err := esClient.Request(context.Background(), followerCreation) + require.NoError(t, err) + defer resp.Body.Close() + }, + }, + test.Step{ + Name: "Check data in the following index in the client cluster", + Test: test.Eventually(func() error { + return elasticsearch.NewDataIntegrityCheck(k, es2Builder).ForIndex(followerIndex).Verify() + }), + }, + es1LicenseTestContext.DeleteEnterpriseLicenseSecret(trialSecretName), + } + } + + test.Sequence(before, stepsFn, es1Builder, es2Builder).RunSequential(t) +} + +func checkRemoteClusterSeed(k *test.K8sClient, clientES, remoteES elasticsearch.Builder, expectedPort int) func() error { + return func() error { + esClient, err := elasticsearch.NewElasticsearchClient(clientES.Elasticsearch, k) + if err != nil { + return err + } + settings, err := esClient.GetRemoteClusterSettings(context.Background()) + if err != nil { + return err + } + persistentSettings := settings.PersistentSettings + if persistentSettings == nil { + return fmt.Errorf("no persistent settings found in client cluster %s/%s", clientES.Elasticsearch.Namespace, clientES.Elasticsearch.Name) + } + cluster, ok := persistentSettings.Cluster.RemoteClusters[remoteES.Name()] + if !ok { + return fmt.Errorf("client cluster %s not found in persistent settings", remoteES.Name()) + } + if len(cluster.Seeds) == 0 { + return fmt.Errorf("no seed for client cluster %s found in persistent settings", remoteES.Name()) + } + expectedSuffix := fmt.Sprintf(":%d", expectedPort) + for _, seed := range cluster.Seeds { + if !strings.HasSuffix(seed, expectedSuffix) { + return fmt.Errorf("client cluster seed must end with %s, found \"%s\"", expectedSuffix, seed) + } + } + return nil + } +} diff --git a/test/e2e/test/elasticsearch/builder.go b/test/e2e/test/elasticsearch/builder.go index 71654a8036..4718284ade 100644 --- a/test/e2e/test/elasticsearch/builder.go +++ b/test/e2e/test/elasticsearch/builder.go @@ -159,6 +159,22 @@ func (b Builder) WithRemoteCluster(remoteEs Builder) Builder { return b } +func (b Builder) WithRemoteClusterAPIKey(remoteEs Builder, apiKey *esv1.RemoteClusterAPIKey) Builder { + b.Elasticsearch.Spec.RemoteClusters = + append(b.Elasticsearch.Spec.RemoteClusters, + esv1.RemoteCluster{ + Name: remoteEs.Ref().Name, + ElasticsearchRef: remoteEs.LocalRef(), + APIKey: apiKey, + }) + return b +} + +func (b Builder) WithRemoteClusterServer() Builder { + b.Elasticsearch.Spec.RemoteClusterServer.Enabled = true + return b +} + func (b Builder) WithNamespace(namespace string) Builder { b.Elasticsearch.ObjectMeta.Namespace = namespace return b diff --git a/test/e2e/test/elasticsearch/checks_data.go b/test/e2e/test/elasticsearch/checks_data.go index 18317fdabd..c4d42c75de 100644 --- a/test/e2e/test/elasticsearch/checks_data.go +++ b/test/e2e/test/elasticsearch/checks_data.go @@ -155,6 +155,10 @@ func (dc *DataIntegrityCheck) Verify() error { if err != nil { return err } + // Check if we have remote cluster failures. + if remoteClusterFailures := results.Cluster.Failures(); len(remoteClusterFailures) > 0 { + return fmt.Errorf("remote cluster failure: %s", remoteClusterFailures) + } // the overall count should be the same if len(results.Hits.Hits) != dc.docCount { return fmt.Errorf("expected %d got %d, data loss", dc.docCount, len(results.Hits.Hits))