Add windows driver installation support

This PR adds windows driver support. It adds a windows base dir to install base yaml files. It also adds a windows alpha kustomization file. To install driver for windows, first set env NODE_OS=windows and GCE_PD_DRIVER_VERSION=alpha and run deploy/kubernetes/deploy-driver.sh script. This PR also reorgnize the dir structure for linux version. Now under overlay, we have a linux and a windows dir. Under each of them, we have alpha, stable, etc. Currently windows only has alpha version.
kubernetes-sigs · May 26, 2020 · 6fa2f52 · 6fa2f52
1 parent ac1f8c0
commit 6fa2f52
Show file tree

Hide file tree

Showing 24 changed files with 398 additions and 58 deletions.
diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node.yaml
@@ -15,21 +15,14 @@ spec:
       # Host network must be used for interaction with Workload Identity in GKE
       # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
       # this requirement when issue is resolved and before any exposure of
-      # metrics ports.
-      hostNetwork: true
+      # metrics ports. But hostNetwork is not working for Windodws, might be an issue
+      # when deploying on GKE windows node. See related issue https://buganizer.corp.google.com/issues/156382455
+      # hostNetwork: true
       priorityClassName: csi-gce-pd-node
       serviceAccountName: csi-gce-pd-node-sa
       containers:
         - name: csi-driver-registrar
           image: gke.gcr.io/csi-node-driver-registrar
-          args:
-            - "--v=5"
-            - "--csi-address=/csi/csi.sock"
-            - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock"
-          lifecycle:
-            preStop:
-              exec:
-                command: ["/bin/sh", "-c", "rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock"]
           env:
             - name: KUBE_NODE_NAME
               valueFrom:
@@ -41,34 +34,11 @@ spec:
             - name: registration-dir
               mountPath: /registration
         - name: gce-pd-driver
-          securityContext:
-            privileged: true
-          # Don't change base image without changing pdImagePlaceholder in
-          # test/k8s-integration/main.go
           image: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
           args:
             - "--v=5"
             - "--endpoint=unix:/csi/csi.sock"
           volumeMounts:
-            - name: kubelet-dir
-              mountPath: /var/lib/kubelet
-              mountPropagation: "Bidirectional"
-            - name: plugin-dir
-              mountPath: /csi
-            - name: device-dir
-              mountPath: /dev
-            # The following mounts are required to trigger host udevadm from
-            # container
-            - name: udev-rules-etc
-              mountPath: /etc/udev
-            - name: udev-rules-lib
-              mountPath: /lib/udev
-            - name: udev-socket
-              mountPath: /run/udev
-            - name: sys
-              mountPath: /sys
-      nodeSelector:
-        kubernetes.io/os: linux
       volumes:
         - name: registration-dir
           hostPath:
@@ -82,28 +52,6 @@ spec:
           hostPath:
             path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/
             type: DirectoryOrCreate
-        - name: device-dir
-          hostPath:
-            path: /dev
-            type: Directory
-        # The following mounts are required to trigger host udevadm from
-        # container
-        - name: udev-rules-etc
-          hostPath:
-            path: /etc/udev
-            type: Directory
-        - name: udev-rules-lib
-          hostPath:
-            path: /lib/udev
-            type: Directory
-        - name: udev-socket
-          hostPath:
-            path: /run/udev
-            type: Directory
-        - name: sys
-          hostPath:
-            path: /sys
-            type: Directory
       # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
       # See "special case". This will tolerate everything. Node component should
       # be scheduled on all nodes.

diff --git a/deploy/kubernetes/delete-driver.sh b/deploy/kubernetes/delete-driver.sh
@@ -13,11 +13,12 @@ set -o errexit
 readonly NAMESPACE="${GCE_PD_DRIVER_NAMESPACE:-gce-pd-csi-driver}"
 readonly DEPLOY_VERSION="${GCE_PD_DRIVER_VERSION:-stable}"
 readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
+readonly OS="${OS:-linux}"
 source "${PKGDIR}/deploy/common.sh"
 
 ensure_kustomize
 
-${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${DEPLOY_VERSION} | ${KUBECTL} delete -v="${VERBOSITY}" --ignore-not-found -f -
+${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${OS}/${DEPLOY_VERSION} | ${KUBECTL} delete -v="${VERBOSITY}" --ignore-not-found -f -
 ${KUBECTL} delete secret cloud-sa -v="${VERBOSITY}" --ignore-not-found
 
 if [[ ${NAMESPACE} != "" && ${NAMESPACE} != "default" ]] && \

diff --git a/deploy/kubernetes/deploy-driver.sh b/deploy/kubernetes/deploy-driver.sh
@@ -19,6 +19,7 @@ set -x
 readonly NAMESPACE="${GCE_PD_DRIVER_NAMESPACE:-gce-pd-csi-driver}"
 readonly DEPLOY_VERSION="${GCE_PD_DRIVER_VERSION:-stable}"
 readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
+readonly OS="${OS:-linux}"
 source "${PKGDIR}/deploy/common.sh"
 
 print_usage()
@@ -51,7 +52,7 @@ function check_service_account()
 	# Grepping for a line with client email returning anything quoted after the colon
 	readonly IAM_NAME=$(grep -Po '"client_email": *\K"[^"]*"' ${GCE_PD_SA_DIR}/cloud-sa.json | tr -d '"')
 	readonly PROJECT=$(grep -Po '"project_id": *\K"[^"]*"' ${GCE_PD_SA_DIR}/cloud-sa.json | tr -d '"')
-	readonly GOTTEN_BIND_ROLES=$(gcloud projects get-iam-policy ${PROJECT} --flatten="bindings[].members" --format='table(bindings.role)' --filter="bindings.members:${IAM_NAME}")
+	readonly GOTTEN_BIND_ROLES=$(gcloud projects get-iam-policy $PROJECT --flatten="bindings[].members" --format='table(bindings.role)' --filter="bindings.members:${IAM_NAME}")
 	readonly BIND_ROLES=$(get_needed_roles)
 	MISSING_ROLES=false
 	for role in ${BIND_ROLES}
@@ -95,6 +96,6 @@ fi
 ${KUBECTL} version
 
 readonly tmp_spec=/tmp/gcp-compute-persistent-disk-csi-driver-specs-generated.yaml
-${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${DEPLOY_VERSION} | tee $tmp_spec
+${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${OS}/${DEPLOY_VERSION} | tee $tmp_spec
 ${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec
 
diff --git a/deploy/kubernetes/overlays/linux/alpha/WARNING.md b/deploy/kubernetes/overlays/linux/alpha/WARNING.md
@@ -0,0 +1,3 @@
+WARNING: DO NOT USE THE ALPHA VERSION OF THE DRIVER FOR PRODUCTION
+
+Alpha features are unsupported and may be unstable and have breaking changes across releases.
diff --git a/deploy/kubernetes/overlays/linux/alpha/kustomization.yaml b/deploy/kubernetes/overlays/linux/alpha/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../stable
+
diff --git a/deploy/kubernetes/overlays/linux/base/enableHostNetwork.yaml b/deploy/kubernetes/overlays/linux/base/enableHostNetwork.yaml
@@ -0,0 +1,8 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-node
+spec:
+  template:
+    spec:
+      hostNetwork: true
diff --git a/deploy/kubernetes/overlays/linux/base/gcepd.yaml b/deploy/kubernetes/overlays/linux/base/gcepd.yaml
@@ -0,0 +1,30 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-node
+spec:
+  template:
+    spec:
+      containers:
+        - name: gce-pd-driver
+          image: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /dev
+            # The following mounts are required to trigger host udevadm from
+            # container
+            - name: udev-rules-etc
+              mountPath: /etc/udev
+            - name: udev-rules-lib
+              mountPath: /lib/udev
+            - name: udev-socket
+              mountPath: /run/udev
+            - name: sys
+              mountPath: /sys
diff --git a/deploy/kubernetes/overlays/linux/base/kustomization.yaml b/deploy/kubernetes/overlays/linux/base/kustomization.yaml
@@ -0,0 +1,30 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../../../base
+patchesStrategicMerge:
+- enableHostNetwork.yaml
+- gcepd.yaml
+- noderegistrar.yaml
+- volumes.yaml
+images:
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  # Don't change stable image without changing pdImagePlaceholder in
+  # test/k8s-integration/main.go
+  newName: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newTag: "v0.7.0-gke.0"
+- name: gke.gcr.io/csi-provisioner
+  newName: gke.gcr.io/csi-provisioner
+  newTag: "v1.5.0-gke.0"
+- name: gke.gcr.io/csi-attacher
+  newName: gke.gcr.io/csi-attacher
+  newTag: "v2.1.1-gke.0"
+- name: gke.gcr.io/csi-node-driver-registrar
+  newName: gke.gcr.io/csi-node-driver-registrar
+  newTag: "v1.2.0-gke.0"
+- name: gke.gcr.io/csi-resizer
+  newName: gke.gcr.io/csi-resizer
+  newTag: "v0.4.0-gke.0"
+- name: gke.gcr.io/csi-snapshotter
+  newName: gke.gcr.io/csi-snapshotter
+  newTag: "v2.1.1-gke.0"
diff --git a/deploy/kubernetes/overlays/linux/base/noderegistrar.yaml b/deploy/kubernetes/overlays/linux/base/noderegistrar.yaml
@@ -0,0 +1,17 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-node
+spec:
+  template:
+    spec:
+      containers:
+        - name: csi-driver-registrar
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock"
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock"]
diff --git a/deploy/kubernetes/overlays/linux/base/volumes.yaml b/deploy/kubernetes/overlays/linux/base/volumes.yaml
@@ -0,0 +1,44 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-node
+spec:
+  template:
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      volumes:
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/
+            type: DirectoryOrCreate
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        # The following mounts are required to trigger host udevadm from
+        # container
+        - name: udev-rules-etc
+          hostPath:
+            path: /etc/udev
+            type: Directory
+        - name: udev-rules-lib
+          hostPath:
+            path: /lib/udev
+            type: Directory
+        - name: udev-socket
+          hostPath:
+            path: /run/udev
+            type: Directory
+        - name: sys
+          hostPath:
+            path: /sys
+            type: Directory
diff --git a/deploy/kubernetes/overlays/linux/dev/WARNING.md b/deploy/kubernetes/overlays/linux/dev/WARNING.md
@@ -0,0 +1,15 @@
+WARNING: DO NOT USE THE STAGING-LATEST VERSION OF THE DRIVER FOR PRODUCTION
+DISCLAIMER: THE LATEST IMAGE IS CONSTANTLY CHANGING WITH DEVELOPMENT AND CAN BE
+BROKEN AT ANY TIME
+
+This is the absolute cutting edge development Driver, it is intended for testing
+and development only and can have vast differences in
+functionality/behavior/configuration. Use only to try the newest features that
+are not guaranteed to work yet.
+
+APPROXIMATE CHANGELOG in latest:
+* Topology
+* RePD
+* Volume ID Format Changed
+* Node ID Format Changed
+* Parameter "zone" Removed
diff --git a/deploy/kubernetes/overlays/linux/dev/controller_always_pull.yaml b/deploy/kubernetes/overlays/linux/dev/controller_always_pull.yaml
@@ -0,0 +1,11 @@
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-controller
+spec:
+  template:
+    spec:
+      containers:
+        - name: gce-pd-driver
+          imagePullPolicy: Always
+
diff --git a/deploy/kubernetes/overlays/linux/dev/kustomization.yaml b/deploy/kubernetes/overlays/linux/dev/kustomization.yaml
@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../alpha
+patches:
+- controller_always_pull.yaml
+- node_always_pull.yaml
+images:
+# Replace this with your private image names and tags
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newName: gcr.io/dyzz-csi-staging/csi/gce-pd-driver
+  newTag: "latest"
+
diff --git a/deploy/kubernetes/overlays/linux/dev/node_always_pull.yaml b/deploy/kubernetes/overlays/linux/dev/node_always_pull.yaml
@@ -0,0 +1,11 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-node
+spec:
+  template:
+    spec:
+      containers:
+        - name: gce-pd-driver
+          imagePullPolicy: Always
+
diff --git a/deploy/kubernetes/overlays/linux/prow-gke-release-staging-head/README.md b/deploy/kubernetes/overlays/linux/prow-gke-release-staging-head/README.md
@@ -0,0 +1 @@
+These overlays are intended to be only used by prow for CI testing.
diff --git a/deploy/kubernetes/overlays/linux/prow-gke-release-staging-head/kustomization.yaml b/deploy/kubernetes/overlays/linux/prow-gke-release-staging-head/kustomization.yaml
@@ -0,0 +1,23 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../base
+images:
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver
+  newTag: "latest"
+- name: gke.gcr.io/csi-provisioner
+  newName: quay.io/k8scsi/csi-provisioner
+  newTag: "canary"
+- name: gke.gcr.io/csi-attacher
+  newName: quay.io/k8scsi/csi-attacher
+  newTag: "canary"
+- name: gke.gcr.io/csi-node-driver-registrar
+  newName: quay.io/k8scsi/csi-node-driver-registrar
+  newTag: "canary"
+- name: gke.gcr.io/csi-resizer
+  newName: quay.io/k8scsi/csi-resizer
+  newTag: "canary"
+- name: gke.gcr.io/csi-snapshotter
+  newName: quay.io/k8scsi/csi-snapshotter
+  newTag: "canary"
diff --git a/deploy/kubernetes/overlays/linux/prow-gke-release-staging-rc/README.md b/deploy/kubernetes/overlays/linux/prow-gke-release-staging-rc/README.md
@@ -0,0 +1 @@
+These overlays are intended to be only used by prow for CI testing.
diff --git a/deploy/kubernetes/overlays/linux/prow-gke-release-staging-rc/enable_sidecar_metrics.yaml b/deploy/kubernetes/overlays/linux/prow-gke-release-staging-rc/enable_sidecar_metrics.yaml
@@ -0,0 +1,19 @@
+# for external-provisioner
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "--metrics-address=:22011"
+
+# for external-attacher
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: "--metrics-address=:22012"
+
+# for external-resizer
+- op: add
+  path: /spec/template/spec/containers/2/args/-
+  value: "--metrics-address=:22013"
+
+# for external-snapshotter
+- op: add
+  path: /spec/template/spec/containers/3/args/-
+  value: "--metrics-address=:22014"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		WARNING: DO NOT USE THE ALPHA VERSION OF THE DRIVER FOR PRODUCTION

		Alpha features are unsupported and may be unstable and have breaking changes across releases.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		These overlays are intended to be only used by prow for CI testing.