Add resize volume functionality to Logstash operator (elastic#7519)

This adds support for expanding Logstash volumes by editing the storage requests
in spec.VolumeClaimTemplate, based on the existing implenentation in Elasticsearch
implemented in elastic#3752.

The same constraints hold -

* Volume size can only be increased, not decreased
* Storage class must specify allowVolumeExpansion: true
* Filesystem resize without pod recreation is only possible if the storage driver allows it

This is based on the Elasticsearch implementation

An update of the storage request in the volumeClaimTemplate will

* Update the storage requests spec of all existing PVCs: they are immediately resized by the storage driver, if inline expansion is supported. Otherwise Pods need to be recreated.
* Delete the StatefulSet, but not the pod that it owns, storing recreation details in an annotation on the owning Logstash resource
* Recreate the StatefulSet with the new volumeClaimTemplate spec
* Remove the recreation annotation from the Logstash resource

This also reworks the webhook validation to validate that storage updates fulfill
the requirements (only increasing storage, using a valid storage class that allows
storage expansion).

This required moving the webhook validation into the controller package to allow use
of the k8sclient without a dependency cycle, and requires some rework in webhook
registration

---------

Co-authored-by: Karen Metts <[email protected]>
Co-authored-by: Peter Brachwitz <[email protected]>

cmd/manager/main.go

@@ -82,6 +82,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/kibana"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/license"
 	licensetrial "github.com/elastic/cloud-on-k8s/v2/pkg/controller/license/trial"
+	lsvalidation "github.com/elastic/cloud-on-k8s/v2/pkg/controller/logstash/validation"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/maps"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/remoteca"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/stackconfigpolicy"
@@ -1011,7 +1012,6 @@ func setupWebhook(
 		&kbv1.Kibana{},
 		&kbv1beta1.Kibana{},
 		&emsv1alpha1.ElasticMapsServer{},
-		&logstashv1alpha1.Logstash{},
 		&policyv1alpha1.StackConfigPolicy{},
 	}
 	for _, obj := range webhookObjects {
@@ -1027,9 +1027,10 @@ func setupWebhook(
 		}
 	}
 
-	// Elasticsearch and ElasticsearchAutoscaling validating webhooks are wired up differently, in order to access the k8s client
+	// Logstash, Elasticsearch and ElasticsearchAutoscaling validating webhooks are wired up differently, in order to access the k8s client
 	esvalidation.RegisterWebhook(mgr, params.ValidateStorageClass, exposedNodeLabels, checker, managedNamespaces)
 	esavalidation.RegisterWebhook(mgr, params.ValidateStorageClass, checker, managedNamespaces)
+	lsvalidation.RegisterWebhook(mgr, params.ValidateStorageClass, managedNamespaces)
 
 	// wait for the secret to be populated in the local filesystem before returning
 	interval := time.Second * 1

config/webhook/manifests.yaml

@@ -202,28 +202,6 @@ webhooks:
     resources:
     - kibanas
   sideEffects: None
-- admissionReviewVersions:
-  - v1
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /validate-logstash-k8s-elastic-co-v1alpha1-logstash
-  failurePolicy: Ignore
-  matchPolicy: Exact
-  name: elastic-logstash-validation-v1alpha1.k8s.elastic.co
-  rules:
-  - apiGroups:
-    - logstash.k8s.elastic.co
-    apiVersions:
-    - v1alpha1
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - logstashes
-  sideEffects: None
 - admissionReviewVersions:
   - v1
   - v1beta1
@@ -312,3 +290,25 @@ webhooks:
     resources:
     - elasticsearchautoscalers
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-logstash-k8s-elastic-co-v1alpha1-logstash
+  failurePolicy: Ignore
+  matchPolicy: Exact
+  name: elastic-logstash-validation-v1alpha1.k8s.elastic.co
+  rules:
+  - apiGroups:
+    - logstash.k8s.elastic.co
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - logstashes
+  sideEffects: None

docs/orchestrating-elastic-stack-applications/logstash.asciidoc

@@ -444,10 +444,18 @@ spec:
 [discrete]
 == Updating the volume claim settings
 
-Changes, such as storage class or volume size, are currently forbidden in `spec.volumeClaimTemplates`.
-To make these changes, you have to fully delete the {ls} resource, delete and recreate or resize the volume, and create a new {ls} resource.
+If the storage class allows link:https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/[volume expansion], you can increase the storage requests size in `spec.volumeClaimTemplates`. 
+ECK updates the existing PersistentVolumeClaims accordingly, and recreates the StatefulSet automatically. 
 
-Before deleting or resizing a persistent queue (PQ) volume, ensure that the queue is empty.
+If the volume driver supports `ExpandInUsePersistentVolumes`, the filesystem is resized online.
+In this case, you do not need to restart the {ls} process or re-create the Pods. 
+
+If the volume driver does not support `ExpandInUsePersistentVolumes`, you must manually delete Pods after the resize so that they can be recreated automatically with the expanded filesystem.
+
+Any other changes in the volumeClaimTemplates--such as changing the storage class or decreasing the volume size--are not allowed. 
+To make changes such as these, you must fully delete the {ls} resource, delete and recreate or resize the volume, and create a new {ls} resource.
+
+Before you delete a persistent queue (PQ) volume, ensure that the queue is empty.
 When using the PQ, we recommend setting `queue.drain: true` on the {ls} Pods to ensure that the queue is drained when Pods are shutdown.
 Note that you should also increase the `terminationGracePeriodSeconds` to a large enough value to allow the queue to drain.
 

pkg/apis/logstash/v1alpha1/labels.go

@@ -6,6 +6,7 @@ package v1alpha1
 
 import (
 	commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1"
+	"github.com/elastic/cloud-on-k8s/v2/pkg/utils/maps"
 )
 
 // GetIdentityLabels will return the common Elastic assigned labels for Logstash
@@ -15,3 +16,10 @@ func (logstash *Logstash) GetIdentityLabels() map[string]string {
 		"logstash.k8s.elastic.co/name": logstash.Name,
 	}
 }
+
+// GetPodIdentityLabels will return the common Elastic assigned labels for a Logstash Pod
+func (logstash *Logstash) GetPodIdentityLabels() map[string]string {
+	return maps.Merge(logstash.GetIdentityLabels(), map[string]string{
+		"logstash.k8s.elastic.co/statefulset-name": Name(logstash.Name),
+	})
+}