Patch annotations onto STS provisioned PVCs

Signed-off-by: Sam McBroom <[email protected]>

.gitignore

@@ -2,6 +2,7 @@
 /.vscode
 /.idea
 /vendor
+.DS_Store
 
 # ignore dependency charts
 charts/*/charts

CHANGELOG.md

@@ -9,6 +9,9 @@ This file itself is based on [Keep a CHANGELOG](https://keepachangelog.com/en/0.
 
 ## [Unreleased]
 
+### Added
+- Add option allowing PVC autoresizer to update annotations of STS provisioned PVCs to match the volumeClaimTemplate ([]())
+
 ## [0.10.0] - 2023-10-13
 
 ### Changed

Makefile

@@ -12,8 +12,9 @@ CONTROLLER_GEN := $(BINDIR)/controller-gen
 GOLANGCI_LINT = $(BINDIR)/golangci-lint
 KUBECTL := $(BINDIR)/kubectl
 KUSTOMIZE := $(BINDIR)/kustomize
+SETUP_ENVTEST := $(BINDIR)/setup-envtest
 
-KUBEBUILDER_ASSETS := $(BINDIR)
+KUBEBUILDER_ASSETS := $(shell $(SETUP_ENVTEST) use -p path $(ENVTEST_K8S_VERSION))
 export KUBEBUILDER_ASSETS
 
 IMAGE_TAG ?= latest
@@ -142,7 +143,6 @@ staticcheck: ## Install staticcheck
 		env GOFLAGS= go install honnef.co/go/tools/cmd/staticcheck@latest; \
 	fi
 
-SETUP_ENVTEST := $(BINDIR)/setup-envtest
 .PHONY: setup-envtest
 setup-envtest: $(SETUP_ENVTEST) ## Download setup-envtest locally if necessary
 $(SETUP_ENVTEST):

README.md

@@ -192,6 +192,35 @@ spec:
 When the size of the largest PVC in the same group is larger than the value set to `resize.topolvm.io/storage_limit` annotation,
 the PVC is resized up to this limit.
 
+#### StatefulSet provisioned PersistentVolumeClaims
+
+PVCs provisioned through a StatefulSet's `volumeClaimTemplates` cannot have their annotations updated automatically. `pvc-autoresizer` can be configured to automatically reconcile PVC annotations to match those found in the owning STS `volumeClaimTemplate` using the `--annotation-patching-enabled` flag. Additionally, this behavior is only enabled if the owning STS has the `resize.topolvm.io/annotation-patching-enabled: "true"` annotation. This reconciliation is only done for the following annotations: `resize.topolvm.io/threshold`, `resize.topolvm.io/inodes-threshold`, `resize.topolvm.io/increase`, `resize.topolvm.io/storage_limit`, and `resize.topolvm.io/initial-resize-group-by`.
+
+For example, for the following existing STS and provisioned PVC:
+
+```yaml
+kind: StatefulSet
+metadata:
+  name: sts
+  annotations:
+    resize.topolvm.io/annotation-patching-enabled: "true"
+spec:
+  volumeClaimTemplates:
+  - kind: PersistentVolumeClaim
+    metadata:
+      annotations:
+        resize.topolvm.io/storage_limit: 20Gi
+# ...
+---
+kind: PersistentVolumeClaim
+metadata:
+  annotations:
+    resize.topolvm.io/storage_limit: 10Gi
+#...
+```
+
+When `pvc-autoresizer` attempts to resize the PVC, it will first update the storage limit annotation value to `20Gi` to match the STS template.
+
 ### Prometheus metrics
 
 ####  `pvcautoresizer_kubernetes_client_fail_total`

charts/pvc-autoresizer/Chart.yaml

@@ -18,18 +18,18 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.13.0
+version: 0.14.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: 0.17.0
+appVersion: 0.18.0
 
 annotations:
   artifacthub.io/images: |
     - name: pvc-autoresizer
-      image: ghcr.io/topolvm/pvc-autoresizer:0.17.0
+      image: ghcr.io/topolvm/pvc-autoresizer:0.18.0
   artifacthub.io/license: Apache-2.0
 
 dependencies:

charts/pvc-autoresizer/README.md

@@ -35,6 +35,7 @@ helm upgrade --create-namespace --namespace pvc-autoresizer -i pvc-autoresizer -
 | controller.affinity | object | `{}` | Affinity for controller deployment. |
 | controller.annotations | object | `{}` | Annotations to be added to controller deployment. |
 | controller.args.additionalArgs | list | `[]` | Specify additional args. |
+| controller.args.annotationPatchingEnabled | bool | `false` | Automatically patch annotations of STS provisioned PVCs to match volumeClaimTemplates. Used as "--annotation-patching-enabled" option |
 | controller.args.interval | string | `"10s"` | Specify interval to monitor pvc capacity. Used as "--interval" option |
 | controller.args.namespaces | list | `[]` | Specify namespaces to control the pvcs of. Empty for all namespaces. Used as "--namespaces" option |
 | controller.args.prometheusURL | string | `"http://prometheus-prometheus-oper-prometheus.prometheus.svc:9090"` | Specify Prometheus URL to query volume stats. Used as "--prometheus-url" option |

charts/pvc-autoresizer/templates/controller/clusterrole.yaml

@@ -58,3 +58,12 @@ rules:
   - list
   - watch
 {{- end }}
+{{- if .Values.controller.args.annotationPatchingEnabled }}
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  verbs:
+  - get
+  - watch
+{{- end }}

charts/pvc-autoresizer/templates/controller/deployment.yaml

@@ -43,6 +43,9 @@ spec:
           {{- if .Values.controller.args.namespaces }}
             - --namespaces={{ join "," .Values.controller.args.namespaces }}
           {{- end }}
+          {{- if .Values.controller.args.annotationPatchingEnabled }}
+            - --annotation-patching-enabled={{ .Values.controller.args.annotationPatchingEnabled }}
+          {{- end }}
           {{- with .Values.controller.args.additionalArgs -}}
             {{ toYaml . | nindent 12 }}
           {{- end }}

charts/pvc-autoresizer/values.yaml

@@ -4,7 +4,7 @@ image:
 
   # image.tag -- pvc-autoresizer image tag to use.
   # @default -- `{{ .Chart.AppVersion }}`
-  tag:  # 0.17.0
+  tag:  # 0.18.0
 
   # image.pullPolicy -- pvc-autoresizer image pullPolicy.
   pullPolicy:  # Always
@@ -30,6 +30,10 @@ controller:
     # Used as "--interval" option
     interval: 10s
 
+    # controller.args.annotationPatchingEnabled -- Automatically patch annotations of STS provisioned PVCs to match volumeClaimTemplates.
+    # Used as "--annotation-patching-enabled" option
+    annotationPatchingEnabled: false
+
     # controller.args.additionalArgs -- Specify additional args.
     additionalArgs: []
 

cmd/main.go

@@ -25,6 +25,7 @@ var config struct {
 	development               bool
 	zapOpts                   zap.Options
 	pvcMutatingWebhookEnabled bool
+	annotationPatchingEnabled bool
 }
 
 // rootCmd represents the base command when called without any subcommands
@@ -61,6 +62,8 @@ func init() {
 	fs.BoolVar(&config.development, "development", false, "Use development logger config")
 	fs.BoolVar(&config.pvcMutatingWebhookEnabled, "pvc-mutating-webhook-enabled", true,
 		"Enable the pvc mutating webhook endpoint")
+	fs.BoolVar(&config.annotationPatchingEnabled, "annotation-patching-enabled", false,
+		"For STS provisioned PVCs, patch annotations in the STS volumeClaimTemplate onto PVCs.")
 
 	goflags := flag.NewFlagSet("zap", flag.ExitOnError)
 	config.zapOpts.BindFlags(goflags)

cmd/run.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/topolvm/pvc-autoresizer/internal/hooks"
 	"github.com/topolvm/pvc-autoresizer/internal/runners"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -78,17 +79,37 @@ func subMain() error {
 		}
 	}
 
+	byObjectCache := map[client.Object]cache.ByObject{
+		&corev1.PersistentVolumeClaim{}: pvcCacheTarget,
+		&storagev1.StorageClass{}:       {},
+	}
+	if config.annotationPatchingEnabled {
+		var stsCacheTarget cache.ByObject
+		if len(config.namespaces) == 0 {
+			stsCacheTarget = cache.ByObject{
+				Namespaces: map[string]cache.Config{
+					cache.AllNamespaces: {},
+				},
+			}
+		} else {
+			stsCacheTarget = cache.ByObject{
+				Namespaces: map[string]cache.Config{},
+			}
+			for _, ns := range config.namespaces {
+				stsCacheTarget.Namespaces[ns] = cache.Config{}
+			}
+		}
+		byObjectCache[&appsv1.StatefulSet{}] = stsCacheTarget
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:        scheme,
 		WebhookServer: webhookServer,
 		Metrics: metricsserver.Options{
 			BindAddress: config.metricsAddr,
 		},
 		Cache: cache.Options{
-			ByObject: map[client.Object]cache.ByObject{
-				&corev1.PersistentVolumeClaim{}: pvcCacheTarget,
-				&storagev1.StorageClass{}:       {},
-			},
+			ByObject: byObjectCache,
 		},
 		HealthProbeBindAddress:  config.healthAddr,
 		LeaderElection:          true,
@@ -134,7 +155,7 @@ func subMain() error {
 
 	pvcAutoresizer := runners.NewPVCAutoresizer(metricsClient, mgr.GetClient(),
 		ctrl.Log.WithName("pvc-autoresizer"),
-		config.watchInterval, mgr.GetEventRecorderFor("pvc-autoresizer"))
+		config.watchInterval, config.annotationPatchingEnabled, mgr.GetEventRecorderFor("pvc-autoresizer"))
 	if err := mgr.Add(pvcAutoresizer); err != nil {
 		setupLog.Error(err, "unable to add autoresier to manager")
 		return err

config/rbac/role.yaml

@@ -23,6 +23,13 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  verbs:
+  - get
+  - watch
 - apiGroups:
   - storage.k8s.io
   resources:

constants.go

@@ -1,25 +1,30 @@
 package pvcautoresizer
 
+const AutoResizerAnnotationPrefix = "resize.topolvm.io/"
+
 // AutoResizeEnabledKey is the key of flag that enables pvc-autoresizer.
-const AutoResizeEnabledKey = "resize.topolvm.io/enabled"
+const AutoResizeEnabledKey = AutoResizerAnnotationPrefix + "enabled"
 
 // ResizeThresholdAnnotation is the key of resize threshold.
-const ResizeThresholdAnnotation = "resize.topolvm.io/threshold"
+const ResizeThresholdAnnotation = AutoResizerAnnotationPrefix + "threshold"
 
 // ResizeInodesThresholdAnnotation is the key of resize threshold for inodes.
-const ResizeInodesThresholdAnnotation = "resize.topolvm.io/inodes-threshold"
+const ResizeInodesThresholdAnnotation = AutoResizerAnnotationPrefix + "inodes-threshold"
 
 // ResizeIncreaseAnnotation is the key of amount increased.
-const ResizeIncreaseAnnotation = "resize.topolvm.io/increase"
+const ResizeIncreaseAnnotation = AutoResizerAnnotationPrefix + "increase"
 
 // StorageLimitAnnotation is the key of storage limit value
-const StorageLimitAnnotation = "resize.topolvm.io/storage_limit"
+const StorageLimitAnnotation = AutoResizerAnnotationPrefix + "storage_limit"
 
 // PreviousCapacityBytesAnnotation is the key of previous volume capacity.
-const PreviousCapacityBytesAnnotation = "resize.topolvm.io/pre_capacity_bytes"
+const PreviousCapacityBytesAnnotation = AutoResizerAnnotationPrefix + "pre_capacity_bytes"
 
 // InitialResizeGroupByAnnotation is the key of the initial-resize group by.
-const InitialResizeGroupByAnnotation = "resize.topolvm.io/initial-resize-group-by"
+const InitialResizeGroupByAnnotation = AutoResizerAnnotationPrefix + "initial-resize-group-by"
+
+// AnnotationPatchingEnabled is the key of flag that enables patching of annotations for STS provisioned PVCs.
+const AnnotationPatchingEnabled = AutoResizerAnnotationPrefix + "annotation-patching-enabled"
 
 // DefaultThreshold is the default value of ResizeThresholdAnnotation.
 const DefaultThreshold = "10%"

internal/metrics/resizer.go

@@ -7,10 +7,12 @@ import (
 
 // Metrics subsystem and all of the keys used by the resizer.
 const (
-	ResizerSuccessResizeTotalKey = "success_resize_total"
-	ResizerFailedResizeTotalKey  = "failed_resize_total"
-	ResizerLoopSecondsTotalKey   = "loop_seconds_total"
-	ResizerLimitReachedTotalKey  = "limit_reached_total"
+	ResizerSuccessResizeTotalKey           = "success_resize_total"
+	ResizerFailedResizeTotalKey            = "failed_resize_total"
+	ResizerLoopSecondsTotalKey             = "loop_seconds_total"
+	ResizerLimitReachedTotalKey            = "limit_reached_total"
+	ResizerSuccessPatchAnnotationsTotalKey = "success_patch_annotations_total"
+	ResizerFailedPatchAnnotationsTotalKey  = "failed_patch_annotations_total"
 )
 
 func init() {
@@ -67,11 +69,39 @@ func (a *resizerLimitReachedTotalAdapter) SpecifyLabels(pvcname string, pvcns st
 	a.metric.With(prometheus.Labels{"persistentvolumeclaim": pvcname, "namespace": pvcns}).Add(0)
 }
 
+type resizerSuccessPatchAnnotationsTotalAdapter struct {
+	metric prometheus.CounterVec
+}
+
+func (a *resizerSuccessPatchAnnotationsTotalAdapter) Increment(pvcName string, pvcNamespace string) {
+	a.metric.With(prometheus.Labels{"persistentvolumeclaim": pvcName, "namespace": pvcNamespace}).Inc()
+}
+
+// SpecifyLabels helps output metrics before the first limit reached event of resize.
+// This method specifies the metric labels and add 0 to the metric value.
+func (a *resizerSuccessPatchAnnotationsTotalAdapter) SpecifyLabels(pvcName string, pvcNamespace string) {
+	a.metric.With(prometheus.Labels{"persistentvolumeclaim": pvcName, "namespace": pvcNamespace}).Add(0)
+}
+
+type resizerFailedPatchAnnotationsTotalAdapter struct {
+	metric prometheus.CounterVec
+}
+
+func (a *resizerFailedPatchAnnotationsTotalAdapter) Increment(pvcName string, pvcNamespace string) {
+	a.metric.With(prometheus.Labels{"persistentvolumeclaim": pvcName, "namespace": pvcNamespace}).Inc()
+}
+
+// SpecifyLabels helps output metrics before the first limit reached event of resize.
+// This method specifies the metric labels and add 0 to the metric value.
+func (a *resizerFailedPatchAnnotationsTotalAdapter) SpecifyLabels(pvcName string, pvcNamespace string) {
+	a.metric.With(prometheus.Labels{"persistentvolumeclaim": pvcName, "namespace": pvcNamespace}).Add(0)
+}
+
 var (
 	resizerSuccessResizeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: MetricsNamespace,
 		Name:      ResizerSuccessResizeTotalKey,
-		Help:      "counter that indicates how many volume expansion processing resized succeed.",
+		Help:      "counter that indicates how many volume expansion processing resizes succeed.",
 	}, []string{"persistentvolumeclaim", "namespace"})
 
 	resizerFailedResizeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
@@ -92,6 +122,18 @@ var (
 		Help:      "counter that indicates how many storage limits were reached.",
 	}, []string{"persistentvolumeclaim", "namespace"})
 
+	resizerSuccessPatchAnnotationsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: MetricsNamespace,
+		Name:      ResizerSuccessPatchAnnotationsTotalKey,
+		Help:      "counter that indicates how many annotation patches on StatefulSet provisioned PersistentVolumeClaims succeed.",
+	}, []string{"persistentvolumeclaim", "namespace"})
+
+	resizerFailedPatchAnnotationsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: MetricsNamespace,
+		Name:      ResizerFailedPatchAnnotationsTotalKey,
+		Help:      "counter that indicates how many annotation patches on StatefulSet provisioned PersistentVolumeClaims fail.",
+	}, []string{"persistentvolumeclaim", "namespace"})
+
 	ResizerSuccessResizeTotal *resizerSuccessResizeTotalAdapter = &resizerSuccessResizeTotalAdapter{
 		metric: *resizerSuccessResizeTotal,
 	}
@@ -104,11 +146,19 @@ var (
 	ResizerLimitReachedTotal *resizerLimitReachedTotalAdapter = &resizerLimitReachedTotalAdapter{
 		metric: *resizerLimitReachedTotal,
 	}
+	ResizerSuccessPatchAnnotationsTotal *resizerSuccessPatchAnnotationsTotalAdapter = &resizerSuccessPatchAnnotationsTotalAdapter{
+		metric: *resizerSuccessPatchAnnotationsTotal,
+	}
+	ResizerFailedPatchAnnotationsTotal *resizerFailedPatchAnnotationsTotalAdapter = &resizerFailedPatchAnnotationsTotalAdapter{
+		metric: *resizerFailedPatchAnnotationsTotal,
+	}
 )
 
 func registerResizerMetrics() {
 	runtimemetrics.Registry.MustRegister(resizerSuccessResizeTotal)
 	runtimemetrics.Registry.MustRegister(resizerFailedResizeTotal)
 	runtimemetrics.Registry.MustRegister(resizerLoopSecondsTotal)
 	runtimemetrics.Registry.MustRegister(resizerLimitReachedTotal)
+	runtimemetrics.Registry.MustRegister(resizerSuccessPatchAnnotationsTotal)
+	runtimemetrics.Registry.MustRegister(resizerFailedPatchAnnotationsTotal)
 }