From a49540fdde4d4add10d5fe75df488523ff0614af Mon Sep 17 00:00:00 2001 From: Guillaume BERNARD Date: Tue, 24 Sep 2024 16:28:08 +0000 Subject: [PATCH 1/4] allow helm chart to accept different extraArgs for nodePlugin and controllerPlugin + documentation --- .../controllerplugin-deployment.yaml | 5 +++++ .../templates/nodeplugin-daemonset.yaml | 5 +++++ charts/cinder-csi-plugin/values.yaml | 2 ++ docs/cinder-csi-plugin/multi-region-clouds.md | 21 +++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml index 542c179f5a..3ee21501a3 100644 --- a/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml +++ b/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml @@ -181,6 +181,11 @@ spec: {{- tpl . $ | trim | nindent 12 }} {{- end }} {{- end }} + {{- if .Values.csi.plugin.controllerPlugin.extraArgs }} + {{- with .Values.csi.plugin.controllerPlugin.extraArgs }} + {{- tpl . $ | trim | nindent 12 }} + {{- end }} + {{- end }} env: - name: CSI_ENDPOINT value: unix://csi/csi.sock diff --git a/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml b/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml index e55e8d69ea..e958de2d32 100644 --- a/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml +++ b/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml @@ -97,6 +97,11 @@ spec: {{- tpl . $ | trim | nindent 12 }} {{- end }} {{- end }} + {{- if .Values.csi.plugin.nodePlugin.extraArgs }} + {{- with .Values.csi.plugin.nodePlugin.extraArgs }} + {{- tpl . $ | trim | nindent 12 }} + {{- end }} + {{- end }} env: - name: CSI_ENDPOINT value: unix://csi/csi.sock diff --git a/charts/cinder-csi-plugin/values.yaml b/charts/cinder-csi-plugin/values.yaml index ec87bb341a..a6af91a266 100644 --- a/charts/cinder-csi-plugin/values.yaml +++ b/charts/cinder-csi-plugin/values.yaml @@ -88,6 +88,7 @@ csi: tolerations: - operator: Exists kubeletDir: /var/lib/kubelet + extraArgs: {} # Allow for specifying internal IP addresses for multiple hostnames # hostAliases: # - ip: "10.0.0.1" @@ -122,6 +123,7 @@ csi: affinity: {} nodeSelector: {} tolerations: [] + extraArgs: {} # Allow for specifying internal IP addresses for multiple hostnames # hostAliases: # - ip: "10.0.0.1" diff --git a/docs/cinder-csi-plugin/multi-region-clouds.md b/docs/cinder-csi-plugin/multi-region-clouds.md index e7e4430631..564df1f3de 100644 --- a/docs/cinder-csi-plugin/multi-region-clouds.md +++ b/docs/cinder-csi-plugin/multi-region-clouds.md @@ -314,3 +314,24 @@ spec: ... ``` +### When using the cinder-csi-plugin helmchart + +When runing the cinder-csi-plugin with multi-region, you need to specify different `extraArgs` on the `cinder-csi-plugin` containers of the deployment and the daemonset. + +When using the helmchart, you need to set the different `extraArgs` on `plugin.nodePlugin.extraArgs` and `plugin.controllerPlugin.extraArgs`. + +If you set the extraArgs in `plugin.extraArgs`, the same `extraArgs` will end up on both the `cinder-csi-plugin` container of both the deployment and the daemonset. + +You will still need to manually create your additionnal daemonsets for your additionnal regions. + +```yaml + nodePlugin: + extraArgs: |- + - --cloud-name=region-one + - --additional-topology + - topology.kubernetes.io/region=region-one + controllerPlugin: + extraArgs: |- + - --cloud-name=region-one + - --cloud-name=region-two +``` From d9ad750398e1b81668893dde7c5eccaca46cb990 Mon Sep 17 00:00:00 2001 From: Guillaume BERNARD Date: Mon, 30 Sep 2024 12:55:42 +0000 Subject: [PATCH 2/4] add extraRbac for resizer and snapshotter --- .../templates/controllerplugin-rbac.yaml | 18 ++++------ charts/cinder-csi-plugin/values.yaml | 14 ++++++++ docs/cinder-csi-plugin/multi-region-clouds.md | 33 ++++++++++++++----- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml index e8d7a909fc..72a4c544ab 100644 --- a/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml +++ b/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml @@ -97,13 +97,6 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - # Secret permission is optional. - # Enable it if your driver needs secret. - # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. - # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. - # - apiGroups: [""] - # resources: ["secrets"] - # verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] @@ -116,6 +109,9 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] + {{- with .Values.csi.snapshotter.extraRbac }} + {{- toYaml . | nindent 2 }} + {{- end }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -135,11 +131,6 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-resizer-role rules: - # The following rule should be uncommented for plugins that require secrets - # for provisioning. - # - apiGroups: [""] - # resources: ["secrets"] - # verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "patch"] @@ -158,6 +149,9 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] + {{- with .Values.csi.resizer.extraRbac }} + {{- toYaml . | nindent 2 }} + {{- end }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/cinder-csi-plugin/values.yaml b/charts/cinder-csi-plugin/values.yaml index a6af91a266..653a5a4603 100644 --- a/charts/cinder-csi-plugin/values.yaml +++ b/charts/cinder-csi-plugin/values.yaml @@ -30,6 +30,14 @@ csi: resources: {} extraArgs: {} extraEnv: [] + # Secret permission is optional. + # Enable it if your driver needs secret. + # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. + # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. + extraRbac: {} + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list"] resizer: image: repository: registry.k8s.io/sig-storage/csi-resizer @@ -38,6 +46,12 @@ csi: resources: {} extraArgs: {} extraEnv: [] + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + extraRbac: {} + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] livenessprobe: image: repository: registry.k8s.io/sig-storage/livenessprobe diff --git a/docs/cinder-csi-plugin/multi-region-clouds.md b/docs/cinder-csi-plugin/multi-region-clouds.md index 564df1f3de..f7d40565c3 100644 --- a/docs/cinder-csi-plugin/multi-region-clouds.md +++ b/docs/cinder-csi-plugin/multi-region-clouds.md @@ -325,13 +325,28 @@ If you set the extraArgs in `plugin.extraArgs`, the same `extraArgs` will end up You will still need to manually create your additionnal daemonsets for your additionnal regions. ```yaml - nodePlugin: - extraArgs: |- - - --cloud-name=region-one - - --additional-topology - - topology.kubernetes.io/region=region-one - controllerPlugin: - extraArgs: |- - - --cloud-name=region-one - - --cloud-name=region-two +nodePlugin: + extraArgs: |- + - --cloud-name=region-one + - --additional-topology + - topology.kubernetes.io/region=region-one +controllerPlugin: + extraArgs: |- + - --cloud-name=region-one + - --cloud-name=region-two ``` + +In addition, if you use the `resizer` and the `snapshotter`, you will need them to be able to read the secrets you defined in the storage class' annotations in order to determine which cloud to address. You will need to add some `extraRbac` in YAML format, like this: + +```yaml +snapshotter: + extraRbac: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +resizer: + extraRbac: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] +``` \ No newline at end of file From c06aa785a7b6663da5b9c1b57d2b2f5a48a6e034 Mon Sep 17 00:00:00 2001 From: Guillaume BERNARD Date: Mon, 30 Sep 2024 13:09:53 +0000 Subject: [PATCH 3/4] bump chart version --- charts/cinder-csi-plugin/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cinder-csi-plugin/Chart.yaml b/charts/cinder-csi-plugin/Chart.yaml index 980f10fb0e..f7c69c31e3 100644 --- a/charts/cinder-csi-plugin/Chart.yaml +++ b/charts/cinder-csi-plugin/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: v1.31.0 description: Cinder CSI Chart for OpenStack name: openstack-cinder-csi -version: 2.31.0 +version: 2.31.1 home: https://github.com/kubernetes/cloud-provider-openstack icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png maintainers: From 29ba8ee1bd9e09c2c711cdc744ae9f2886390ab1 Mon Sep 17 00:00:00 2001 From: Guillaume BERNARD Date: Tue, 15 Oct 2024 10:30:41 +0000 Subject: [PATCH 4/4] bump chart version --- charts/cinder-csi-plugin/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cinder-csi-plugin/Chart.yaml b/charts/cinder-csi-plugin/Chart.yaml index f7c69c31e3..c21b962414 100644 --- a/charts/cinder-csi-plugin/Chart.yaml +++ b/charts/cinder-csi-plugin/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: v1.31.0 description: Cinder CSI Chart for OpenStack name: openstack-cinder-csi -version: 2.31.1 +version: 2.31.3 home: https://github.com/kubernetes/cloud-provider-openstack icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png maintainers: