Add windows driver installation support

This PR adds windows driver support. It adds a windows base dir to install base yaml files. It also adds a windows alpha kustomization file. To install driver for windows, first set env NODE_OS=windows and GCE_PD_DRIVER_VERSION=alpha and run deploy/kubernetes/deploy-driver.sh script. This PR also reorgnize the dir structure for linux version. Now under overlay, we have a linux and a windows dir. Under each of them, we have alpha, stable, etc. Currently windows only has alpha version.
kubernetes-sigs · Jun 4, 2020 · 6314bf5 · 6314bf5
1 parent 85088c4
commit 6314bf5
Show file tree

Hide file tree

Showing 28 changed files with 289 additions and 36 deletions.
diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml
@@ -17,6 +17,8 @@ spec:
       # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
       # this requirement when issue is resolved and before any exposure of
       # metrics ports
+      nodeSelector:
+        kubernetes.io/os: linux
       hostNetwork: true
       serviceAccountName: csi-gce-pd-controller-sa
       priorityClassName: csi-gce-pd-controller
@@ -82,5 +84,10 @@ spec:
         - name: cloud-sa-volume
           secret:
             secretName: cloud-sa
+      # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+      # See "special case". This will tolerate everything. Node component should
+      # be scheduled on all nodes.
+      tolerations:
+      - operator: Exists
   # This is needed due to https://github.com/kubernetes-sigs/kustomize/issues/504
   volumeClaimTemplates: []
diff --git a/deploy/kubernetes/base/kustomization.yaml b/deploy/kubernetes/base/kustomization.yaml
@@ -3,6 +3,5 @@ commonLabels:
 namespace:
   gce-pd-csi-driver
 resources:
-- node.yaml
 - controller.yaml
 - setup-cluster.yaml
diff --git a/deploy/kubernetes/base/setup-cluster.yaml b/deploy/kubernetes/base/setup-cluster.yaml
@@ -164,15 +164,6 @@ spec:
   volumes:
   - '*'
   hostNetwork: true
-  allowedHostPaths:
-  - pathPrefix: "/var/lib/kubelet/plugins_registry/"
-  - pathPrefix: "/var/lib/kubelet"
-  - pathPrefix: "/var/lib/kubelet/plugins/pd.csi.storage.gke.io/"
-  - pathPrefix: "/dev"
-  - pathPrefix: "/etc/udev"
-  - pathPrefix: "/lib/udev"
-  - pathPrefix: "/run/udev"
-  - pathPrefix: "/sys"
 ---
 
 kind: ClusterRole
@@ -199,6 +190,19 @@ subjects:
 - kind: ServiceAccount
   name: csi-gce-pd-node-sa
 
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: csi-gce-pd-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-node-deploy
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

diff --git a/deploy/kubernetes/delete-driver.sh b/deploy/kubernetes/delete-driver.sh
@@ -13,11 +13,16 @@ set -o errexit
 readonly NAMESPACE="${GCE_PD_DRIVER_NAMESPACE:-gce-pd-csi-driver}"
 readonly DEPLOY_VERSION="${GCE_PD_DRIVER_VERSION:-stable}"
 readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
+readonly DEPLOY_OS_VERSIONS=${DEPLOY_OS_VERSIONS:-"linux stable"}
 source "${PKGDIR}/deploy/common.sh"
 
 ensure_kustomize
 
-${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${DEPLOY_VERSION} | ${KUBECTL} delete -v="${VERBOSITY}" --ignore-not-found -f -
+echo ${DEPLOY_OS_VERSIONS} | tr ';' '\n' | while read -r NODE_OS VERSION; do \
+  VERSION="${VERSION:-${DEPLOY_VERSION}}"
+	${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${NODE_OS}/${VERSION} | ${KUBECTL} delete -v="${VERBOSITY}" --ignore-not-found -f -; \
+done
+
 ${KUBECTL} delete secret cloud-sa -v="${VERBOSITY}" --ignore-not-found
 
 if [[ ${NAMESPACE} != "" && ${NAMESPACE} != "default" ]] && \

diff --git a/deploy/kubernetes/deploy-driver.sh b/deploy/kubernetes/deploy-driver.sh
@@ -19,6 +19,7 @@ set -x
 readonly NAMESPACE="${GCE_PD_DRIVER_NAMESPACE:-gce-pd-csi-driver}"
 readonly DEPLOY_VERSION="${GCE_PD_DRIVER_VERSION:-stable}"
 readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
+readonly DEPLOY_OS_VERSIONS=${DEPLOY_OS_VERSIONS:-"linux stable"}
 source "${PKGDIR}/deploy/common.sh"
 
 print_usage()
@@ -94,7 +95,10 @@ fi
 # Debug log: print ${KUBECTL} version
 ${KUBECTL} version
 
-readonly tmp_spec=/tmp/gcp-compute-persistent-disk-csi-driver-specs-generated.yaml
-${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${DEPLOY_VERSION} | tee $tmp_spec
-${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec
+echo ${DEPLOY_OS_VERSIONS} | tr ';' '\n' | while read -r NODE_OS VERSION; do \
+	VERSION="${VERSION:-${DEPLOY_VERSION}}"; \
+	tmp_spec=/tmp/gcp-compute-persistent-disk-csi-driver-specs-generated-${NODE_OS}.yaml; \
+	${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${NODE_OS}/${VERSION} | tee $tmp_spec; \
+	${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec; \
+done
 
diff --git a/deploy/kubernetes/overlays/alpha/WARNING.md → ...ubernetes/overlays/linux/alpha/WARNING.md b/deploy/kubernetes/overlays/alpha/WARNING.md → ...ubernetes/overlays/linux/alpha/WARNING.md
diff --git a/...ernetes/overlays/alpha/kustomization.yaml → ...s/overlays/linux/alpha/kustomization.yaml b/...ernetes/overlays/alpha/kustomization.yaml → ...s/overlays/linux/alpha/kustomization.yaml
diff --git a/deploy/kubernetes/overlays/linux/base/allowedHostPaths.yaml b/deploy/kubernetes/overlays/linux/base/allowedHostPaths.yaml
@@ -0,0 +1,14 @@
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: csi-gce-pd-node-psp
+spec:
+  allowedHostPaths:
+  - pathPrefix: "/var/lib/kubelet/plugins_registry/"
+  - pathPrefix: "/var/lib/kubelet"
+  - pathPrefix: "/var/lib/kubelet/plugins/pd.csi.storage.gke.io/"
+  - pathPrefix: "/dev"
+  - pathPrefix: "/etc/udev"
+  - pathPrefix: "/lib/udev"
+  - pathPrefix: "/run/udev"
+  - pathPrefix: "/sys"
diff --git a/deploy/kubernetes/overlays/linux/base/enableHostNetwork.yaml b/deploy/kubernetes/overlays/linux/base/enableHostNetwork.yaml
@@ -0,0 +1,8 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-node
+spec:
+  template:
+    spec:
+      hostNetwork: true
diff --git a/deploy/kubernetes/overlays/linux/base/kustomization.yaml b/deploy/kubernetes/overlays/linux/base/kustomization.yaml
@@ -0,0 +1,31 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../../../base
+namespace:
+  gce-pd-csi-driver
+resources:
+- node.yaml
+patchesStrategicMerge:
+- allowedHostPaths.yaml
+images:
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  # Don't change stable image without changing pdImagePlaceholder in
+  # test/k8s-integration/main.go
+  newName: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newTag: "v0.7.0-gke.0"
+- name: gke.gcr.io/csi-provisioner
+  newName: gke.gcr.io/csi-provisioner
+  newTag: "v1.5.0-gke.0"
+- name: gke.gcr.io/csi-attacher
+  newName: gke.gcr.io/csi-attacher
+  newTag: "v2.1.1-gke.0"
+- name: gke.gcr.io/csi-node-driver-registrar
+  newName: gke.gcr.io/csi-node-driver-registrar
+  newTag: "v1.2.0-gke.0"
+- name: gke.gcr.io/csi-resizer
+  newName: gke.gcr.io/csi-resizer
+  newTag: "v0.4.0-gke.0"
+- name: gke.gcr.io/csi-snapshotter
+  newName: gke.gcr.io/csi-snapshotter
+  newTag: "v2.1.1-gke.0"
diff --git a/deploy/kubernetes/base/node.yaml → .../kubernetes/overlays/linux/base/node.yaml b/deploy/kubernetes/base/node.yaml → .../kubernetes/overlays/linux/base/node.yaml
@@ -15,21 +15,20 @@ spec:
       # Host network must be used for interaction with Workload Identity in GKE
       # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
       # this requirement when issue is resolved and before any exposure of
-      # metrics ports.
+      # metrics ports. But hostNetwork is not working for Windows, might be an issue
+      # when deploying on GKE Windows node.
       hostNetwork: true
       priorityClassName: csi-gce-pd-node
       serviceAccountName: csi-gce-pd-node-sa
+      nodeSelector:
+        kubernetes.io/os: linux
       containers:
         - name: csi-driver-registrar
           image: gke.gcr.io/csi-node-driver-registrar
           args:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"
             - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock"
-          lifecycle:
-            preStop:
-              exec:
-                command: ["/bin/sh", "-c", "rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock"]
           env:
             - name: KUBE_NODE_NAME
               valueFrom:
@@ -41,14 +40,14 @@ spec:
             - name: registration-dir
               mountPath: /registration
         - name: gce-pd-driver
-          securityContext:
-            privileged: true
           # Don't change base image without changing pdImagePlaceholder in
           # test/k8s-integration/main.go
           image: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
           args:
             - "--v=5"
             - "--endpoint=unix:/csi/csi.sock"
+          securityContext:
+            privileged: true
           volumeMounts:
             - name: kubelet-dir
               mountPath: /var/lib/kubelet
@@ -67,8 +66,6 @@ spec:
               mountPath: /run/udev
             - name: sys
               mountPath: /sys
-      nodeSelector:
-        kubernetes.io/os: linux
       volumes:
         - name: registration-dir
           hostPath:
@@ -108,4 +105,4 @@ spec:
       # See "special case". This will tolerate everything. Node component should
       # be scheduled on all nodes.
       tolerations:
-      - operator: Exists
+      - operator: Exists
diff --git a/deploy/kubernetes/overlays/linux/base/noderegistrar.yaml b/deploy/kubernetes/overlays/linux/base/noderegistrar.yaml
@@ -0,0 +1,13 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-node
+spec:
+  template:
+    spec:
+      containers:
+        - name: csi-driver-registrar
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock"
diff --git a/deploy/kubernetes/overlays/dev/WARNING.md → .../kubernetes/overlays/linux/dev/WARNING.md b/deploy/kubernetes/overlays/dev/WARNING.md → .../kubernetes/overlays/linux/dev/WARNING.md
@@ -5,11 +5,4 @@ BROKEN AT ANY TIME
 This is the absolute cutting edge development Driver, it is intended for testing
 and development only and can have vast differences in
 functionality/behavior/configuration. Use only to try the newest features that
-are not guaranteed to work yet.
-
-APPROXIMATE CHANGELOG in latest:
-* Topology
-* RePD
-* Volume ID Format Changed
-* Node ID Format Changed
-* Parameter "zone" Removed
+are not guaranteed to work yet.
diff --git a/.../overlays/dev/controller_always_pull.yaml → ...ays/linux/dev/controller_always_pull.yaml b/.../overlays/dev/controller_always_pull.yaml → ...ays/linux/dev/controller_always_pull.yaml
diff --git a/...ubernetes/overlays/dev/kustomization.yaml → ...tes/overlays/linux/dev/kustomization.yaml b/...ubernetes/overlays/dev/kustomization.yaml → ...tes/overlays/linux/dev/kustomization.yaml
diff --git a/...rnetes/overlays/dev/node_always_pull.yaml → .../overlays/linux/dev/node_always_pull.yaml b/...rnetes/overlays/dev/node_always_pull.yaml → .../overlays/linux/dev/node_always_pull.yaml
diff --git a/...s/prow-gke-release-staging-head/README.md → ...x/prow-gke-release-staging-head/README.md b/...s/prow-gke-release-staging-head/README.md → ...x/prow-gke-release-staging-head/README.md
diff --git a/...e-release-staging-head/kustomization.yaml → ...e-release-staging-head/kustomization.yaml b/...e-release-staging-head/kustomization.yaml → ...e-release-staging-head/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- ../../base
+- ../base
 images:
 - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
   newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver

diff --git a/...ays/prow-gke-release-staging-rc/README.md → ...nux/prow-gke-release-staging-rc/README.md b/...ays/prow-gke-release-staging-rc/README.md → ...nux/prow-gke-release-staging-rc/README.md
diff --git a/...gke-release-staging-rc/kustomization.yaml → ...gke-release-staging-rc/kustomization.yaml b/...gke-release-staging-rc/kustomization.yaml → ...gke-release-staging-rc/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- ../../base
+- ../base
 images:
 - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
   newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver

diff --git a/...rnetes/overlays/stable/kustomization.yaml → .../overlays/linux/stable/kustomization.yaml b/...rnetes/overlays/stable/kustomization.yaml → .../overlays/linux/stable/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- ../../base
+- ../base
 images:
 - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
   # Don't change stable image without changing pdImagePlaceholder in

diff --git a/deploy/kubernetes/overlays/windows/base/allowedHostPaths.yaml b/deploy/kubernetes/overlays/windows/base/allowedHostPaths.yaml
@@ -0,0 +1,12 @@
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: csi-gce-pd-node-psp
+spec:
+  allowedHostPaths:
+  - pathPrefix: \var\lib\kubelet
+  - pathPrefix: \var\lib\kubelet\plugins_registry
+  - pathPrefix: \var\lib\kubelet\plugins\pd.csi.storage.gke.io
+  - pathPrefix: \\.\pipe\csi-proxy-disk-v1alpha1
+  - pathPrefix: \\.\pipe\csi-proxy-volume-v1alpha1
+  - pathPrefix: \\.\pipe\csi-proxy-filesystem-v1alpha1
diff --git a/deploy/kubernetes/overlays/windows/base/kustomization.yaml b/deploy/kubernetes/overlays/windows/base/kustomization.yaml
@@ -0,0 +1,36 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../../../base
+namespace:
+  gce-pd-csi-driver
+resources:
+- node.yaml
+patchesStrategicMerge:
+- allowedHostPaths.yaml
+images:
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  # Don't change stable image without changing pdImagePlaceholder in
+  # test/k8s-integration/main.go
+  newName: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newTag: "v0.7.0-gke.0"
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver-win
+  # Temporarly set to the private repo. Will swtich to public one
+  # once it is available.
+  newName: gcr.io/jing-k8s-dev/gce-pd-windows-2019
+  newTag: "0.2.0"
+- name: gke.gcr.io/csi-provisioner
+  newName: gke.gcr.io/csi-provisioner
+  newTag: "v1.5.0-gke.0"
+- name: gke.gcr.io/csi-attacher
+  newName: gke.gcr.io/csi-attacher
+  newTag: "v2.1.1-gke.0"
+- name: gke.gcr.io/csi-node-driver-registrar
+  newName: gcr.io/k8s-staging-csi/csi-node-driver-registrar
+  newTag: "amd64-windows-v20200428-v1.3.0-26-g510710d5"
+- name: gke.gcr.io/csi-resizer
+  newName: gke.gcr.io/csi-resizer
+  newTag: "v0.4.0-gke.0"
+- name: gke.gcr.io/csi-snapshotter
+  newName: gke.gcr.io/csi-snapshotter
+  newTag: "v2.1.1-gke.0"