From 8cf3d52fa43bac64b0624b7f00b4551bec8f5050 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 22 Jun 2020 13:10:18 +0200 Subject: [PATCH] enable Prometheus, gather CSI call counts + PMEM stats The Prometheus integration uses the approach from https://github.com/helm/charts/issues/22899: - HTTP for metrics endpoints - container ports tell Prometheus which containers to scrape and how CSI call counts are the same as in the sidecars. This enables correlating statistics and ensures that also node-local operations are captured; kubelet doesn't seem to be instrumented. Internal communication is instrumented the same way. PMEM usage statistics are recorded by querying the active device manager each time the metrics data gets scraped. The metrics support is enabled unconditionally in the operator and all pre-generated deployment files and use plain HTTP for the sake of simplicity. This is based on the rationale that the data itself is not sensitive and should always be readily available if desired. --- deploy/bindata_generated.go | 32 +-- deploy/kubernetes-1.17/direct/pmem-csi.yaml | 17 +- .../direct/testing/pmem-csi.yaml | 17 +- deploy/kubernetes-1.17/lvm/pmem-csi.yaml | 17 +- .../kubernetes-1.17/lvm/testing/pmem-csi.yaml | 17 +- .../pmem-csi-direct-testing.yaml | 17 +- deploy/kubernetes-1.17/pmem-csi-direct.yaml | 17 +- .../kubernetes-1.17/pmem-csi-lvm-testing.yaml | 17 +- deploy/kubernetes-1.17/pmem-csi-lvm.yaml | 17 +- deploy/kubernetes-1.18/direct/pmem-csi.yaml | 17 +- .../direct/testing/pmem-csi.yaml | 17 +- deploy/kubernetes-1.18/lvm/pmem-csi.yaml | 17 +- .../kubernetes-1.18/lvm/testing/pmem-csi.yaml | 17 +- .../pmem-csi-direct-testing.yaml | 17 +- deploy/kubernetes-1.18/pmem-csi-direct.yaml | 17 +- .../kubernetes-1.18/pmem-csi-lvm-testing.yaml | 17 +- deploy/kubernetes-1.18/pmem-csi-lvm.yaml | 17 +- .../direct/pmem-csi.yaml | 17 +- .../direct/testing/pmem-csi.yaml | 17 +- .../kubernetes-1.19-alpha/lvm/pmem-csi.yaml | 17 +- .../lvm/testing/pmem-csi.yaml | 17 +- .../pmem-csi-direct-testing.yaml | 17 +- .../pmem-csi-direct.yaml | 17 +- .../pmem-csi-lvm-testing.yaml | 17 +- .../kubernetes-1.19-alpha/pmem-csi-lvm.yaml | 17 +- deploy/kubernetes-1.19/direct/pmem-csi.yaml | 17 +- .../direct/testing/pmem-csi.yaml | 17 +- deploy/kubernetes-1.19/lvm/pmem-csi.yaml | 17 +- .../kubernetes-1.19/lvm/testing/pmem-csi.yaml | 17 +- .../pmem-csi-direct-testing.yaml | 17 +- deploy/kubernetes-1.19/pmem-csi-direct.yaml | 17 +- .../kubernetes-1.19/pmem-csi-lvm-testing.yaml | 17 +- deploy/kubernetes-1.19/pmem-csi-lvm.yaml | 17 +- deploy/kustomize/driver/pmem-csi.yaml | 8 +- .../kubernetes-base-direct/kustomization.yaml | 4 +- .../kubernetes-base-lvm/kustomization.yaml | 4 +- .../README.md | 0 .../driverinfo-beta.yaml | 0 .../kustomization.yaml | 0 .../rbac/README.md | 0 .../rbac/kustomization.yaml | 0 .../kustomization.yaml | 19 ++ .../kustomize/patches/metrics-controller.yaml | 30 +++ deploy/kustomize/patches/metrics-node.yaml | 18 ++ deploy/prometheus.yaml | 42 ++++ docs/DEVELOPMENT.md | 13 +- docs/install.md | 167 ++++++++++++- go.mod | 6 +- go.sum | 15 +- pkg/grpc-server/server.go | 5 +- pkg/pmem-csi-driver/controllerserver-node.go | 9 +- pkg/pmem-csi-driver/main.go | 21 +- pkg/pmem-csi-driver/pmem-csi-driver.go | 234 +++++++++++++----- pkg/pmem-csi-driver/pmem-csi-driver_test.go | 32 +-- .../deployment/controller_driver.go | 32 ++- pkg/pmem-device-manager/pmd-lvm.go | 40 +-- pkg/pmem-device-manager/pmd-manager.go | 19 +- pkg/pmem-device-manager/pmd-ndctl.go | 40 ++- pkg/pmem-grpc/grpc.go | 13 +- pkg/registryserver/registryserver.go | 48 +++- pkg/registryserver/registryserver_test.go | 4 +- test/e2e/deploy/deploy.go | 2 +- test/e2e/metrics/metrics.go | 88 ++++++- test/e2e/operator/validate/validate.go | 2 + test/e2e/tls/tls.go | 5 +- 65 files changed, 1281 insertions(+), 215 deletions(-) rename deploy/kustomize/{kubernetes-base => kubernetes-no-metrics}/README.md (100%) rename deploy/kustomize/{kubernetes-base => kubernetes-no-metrics}/driverinfo-beta.yaml (100%) rename deploy/kustomize/{kubernetes-base => kubernetes-no-metrics}/kustomization.yaml (100%) rename deploy/kustomize/{kubernetes-base => kubernetes-no-metrics}/rbac/README.md (100%) rename deploy/kustomize/{kubernetes-base => kubernetes-no-metrics}/rbac/kustomization.yaml (100%) create mode 100644 deploy/kustomize/kubernetes-with-metrics/kustomization.yaml create mode 100644 deploy/kustomize/patches/metrics-controller.yaml create mode 100644 deploy/kustomize/patches/metrics-node.yaml create mode 100644 deploy/prometheus.yaml diff --git a/deploy/bindata_generated.go b/deploy/bindata_generated.go index 79df384623..83af9195c0 100644 --- a/deploy/bindata_generated.go +++ b/deploy/bindata_generated.go @@ -84,7 +84,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _deployKubernetes117DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5f\x6f\xe3\xb8\x11\x7f\xf7\xa7\x60\xb7\xf7\x70\x07\x54\x56\x72\x45\x81\x83\x00\x3f\x64\x37\xbe\xad\xd1\x8d\x63\x24\xb9\x7b\x0d\x68\x6a\x2c\xb3\xa6\x48\x1e\x39\xf2\x46\x57\xf4\xbb\x17\xa4\x24\x5b\xff\x2c\xff\xb9\x64\xd3\x73\x1e\x22\x8b\x1c\xce\x8f\x33\xc3\x99\xdf\x48\xfe\x2b\xf9\x0c\x12\x0c\x45\x88\xc9\x57\x8e\x6b\xf2\x21\xa5\x1b\x20\x9b\xcc\xa2\x4a\xf9\xef\xf0\xe1\x6f\x24\x56\x44\x2a\x24\x10\x73\xfc\xcb\x68\x44\x35\xff\x15\x8c\xe5\x4a\x46\x64\x7b\x3d\xda\x70\x19\x47\xe4\x11\xcc\x96\x33\xb8\x61\x4c\x65\x12\x47\x29\x20\x8d\x29\xd2\x68\x44\x88\xa0\x4b\x10\xd6\x5d\x11\xa2\x53\x48\x03\x66\xf9\x98\x4b\x04\x31\x66\x2a\x0d\x63\xd0\x42\xe5\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x22\x44\xd2\x14\xa2\x9d\x60\xc0\x94\x44\xa3\x84\x00\x53\x8e\x59\x4d\x19\x44\x24\x86\x15\xcd\x04\x8e\x82\x20\x68\x20\x34\x4b\xca\xc6\x34\xc3\xb5\x32\xfc\x77\xea\xd6\x1c\x6f\x7e\xb2\x63\xae\xc2\x1d\xf6\x07\x25\xe0\x0d\x11\xc3\x0b\x82\x91\x54\xb8\x49\x5b\xee\x50\x81\x09\xd8\x2a\xe9\xc7\x6f\x32\x01\x36\x1a\x05\x84\x6a\xfe\xd9\xa8\x4c\x7b\x1c\x01\xf9\xf0\x61\x44\x88\x01\xab\x32\xc3\xa0\xbc\x07\x32\xd6\x8a\x4b\xb4\x23\x42\xb6\x60\x96\xe5\xed\x04\xd0\xff\xff\x4a\x91\xad\xfd\x95\xe0\xb6\xb8\x15\x83\x00\x04\x7f\x99\xe9\x98\x96\x97\xcc\x80\xbb\xec\xe8\x64\x4a\x99\x98\xcb\xba\xd5\xba\x20\x04\x50\x0b\x6f\x85\xc0\xa2\x32\x34\x81\x83\xca\x99\xe5\xe5\x14\x46\x35\x65\x1c\xf9\x21\x28\x3b\x00\x7b\x4c\xa5\xd2\x16\x14\xbd\x1b\x2f\xa1\x9e\xe4\x0a\xad\xe2\x1e\xc5\x1d\x51\xaa\xb5\xed\x0a\x1b\xd0\x82\x33\x6a\xa1\xcf\x93\x17\x04\xf4\x27\x91\x59\x04\xf3\x1e\x71\x6d\x32\x29\xc1\x9c\x15\xc5\xda\xed\xcc\x22\x48\xdc\x2a\x91\xa5\x97\x39\xf0\x2c\x5f\xb5\x14\x32\x41\x79\x7a\xba\xd6\x32\x56\xce\x8e\xd5\x2a\x50\x05\xb5\x07\x0f\x4c\x4b\xdd\x69\x59\x60\x0b\x9d\x14\x70\x66\xb4\x77\xf7\x22\xa9\xb6\x6b\x85\xe3\x63\x9b\x2a\x5d\x56\x4e\x1f\xda\xd5\x6b\xe9\x70\x05\xa0\xbb\xe1\x23\xaa\x8e\xa7\x11\xa9\xe2\xd7\x74\xca\x1f\x58\xee\x34\x73\x50\x44\xca\xd6\xe9\x11\x4b\xd4\xd4\x5c\x56\x17\x3f\x72\x19\x73\x99\xbc\x65\x41\xb7\xbc\x99\x41\x94\x80\x81\xf2\xa8\x04\x3c\xc0\xca\x29\xaf\xcc\x36\xb0\x97\x11\x21\xb5\x12\x7f\x7a\x65\xb6\xd9\xf2\xdf\xc0\xd0\x67\xb0\x5e\x7e\xf3\x2d\x68\x49\x2d\x8b\xbf\x8f\x17\x2e\x35\x76\xbd\xfc\x9c\x55\x35\xde\xde\xec\x6d\xbe\xfa\x8e\x44\xd5\x6a\x60\x4e\x97\x56\x06\x77\x24\xc2\x60\x44\xae\xaf\xae\xae\xae\x3c\x08\xa4\x26\x01\x5c\x34\xee\x5a\x10\xc0\x50\x99\x02\x26\xd5\xfa\x90\xca\x0b\xf6\xf0\x8e\xd6\x4a\x01\x0d\x67\xf6\x7c\x53\x5d\xf7\x9a\xea\xfa\xad\x4d\x45\x08\xe6\x1a\x22\x32\x57\x31\x38\xad\x1d\xdb\x39\xa6\xb7\x3f\xcb\x8f\x48\x11\x56\x99\x78\x84\xf7\xec\x8d\x2a\x3b\x56\x7c\x33\x22\xd7\x1d\x33\xa5\xae\x5a\x7c\xa9\xc1\x3a\x62\xb9\x8b\x70\xdb\x22\x9e\xe6\x03\xf0\x11\x52\x2d\x28\x42\x89\xaa\x66\x33\xf7\x11\x0d\x80\x47\x21\x5e\x04\xf2\xa0\xe0\x57\x58\xae\x95\xda\x44\x84\x27\x52\x19\xf0\x53\x2b\xd3\x7a\x30\xab\x15\x97\x1c\xf3\x3d\x3c\xc7\x06\x6e\x3a\x77\x9d\x23\x7e\xcb\xb8\x81\xf8\x36\x33\x5c\x26\x8f\x6c\x0d\x71\x26\xb8\x4c\x66\x7e\xe1\xf2\xf6\xf4\x05\x58\xe6\x30\xd5\x25\x8b\x35\x1f\x4b\xd7\x3d\x81\x49\x6d\x73\x38\x28\x3c\x39\x7d\xd1\x06\xac\x0b\xc9\xd6\xb8\x9b\xb1\x81\x3c\xea\xdb\x60\x8f\xfd\xaa\x8f\xd2\x60\xa8\x0b\x16\x32\x57\x38\x93\x9d\xf1\x2d\x15\x19\x74\x54\x79\x96\x24\xd5\x87\xbe\xdb\x2b\x2a\x2c\x54\x23\x4e\x33\xe5\x12\xcc\x6e\x09\xd7\x85\xa6\x29\x95\xf1\x7e\xcd\x80\x84\x99\x35\xa1\x50\x8c\x8a\x70\xc9\x65\xb8\x73\x7c\x6c\xf8\xb6\x06\x3a\x20\xc1\x76\xf2\xf7\xfa\xd7\x54\xc5\x30\xe9\xd9\x5e\x40\x82\xaa\x99\x9e\x64\x92\xbf\x44\x61\x18\x32\xcb\xc3\x66\x34\x8d\xad\x62\x9b\xba\x8c\x81\x84\x5b\x34\xf9\xb4\x92\x45\xa6\xa3\x30\xbc\x1a\xfb\xbf\x68\x9f\xc9\x2b\xf5\x45\x96\xfb\xe2\xbb\x8f\x49\xb4\x4f\x5f\xe5\xb8\xf3\x29\x8f\x27\xdf\x7d\xff\xaf\x5f\x3e\x4e\x9f\xe7\xf7\xb7\xd3\xe7\xf9\xcd\xdd\xf4\x87\xfa\x1c\x46\x7f\xe6\x02\x26\x21\x03\x83\x36\x64\x74\xcc\x0c\x36\xc6\xc1\x60\x7d\x06\x0a\xdb\x9e\xb2\x81\xbc\x3d\x63\x03\x79\x7d\x46\x61\x48\x97\x42\x26\xdf\x7d\xbf\xb8\x9b\xde\x3d\x7f\x7a\x9c\x3d\xdf\x3e\xcc\x7e\x9d\x3e\xb4\x20\x81\xdc\xd6\x5d\x53\xa4\xa4\x26\xfe\x51\x2b\x3e\x7e\x36\x2a\x6d\x86\xc8\x8a\x83\x88\x4b\xaa\x51\xff\xb4\xaa\x51\x73\xd0\x0b\x2d\x28\xae\x23\x7f\xfc\xc6\xce\x7a\x2e\xa5\x74\xd0\x3c\x4d\x1f\xee\x66\xf3\x9b\xa7\xd9\xfd\xfc\xf9\xcb\xfd\xe7\xe7\xc5\xcd\xd3\x3f\xdb\x98\x22\x12\x62\xaa\x43\x04\x93\x96\x4f\x3c\x02\xa1\x92\xce\x52\x7d\xb6\xe8\x2e\xd5\x3d\x53\xbb\x39\x3c\xa5\x09\x44\xc4\x0f\xb4\x03\x37\x62\x54\x52\x93\x37\xe7\x2e\x32\x21\x16\x4a\x70\x96\x47\x64\xb6\x9a\x2b\x5c\x18\xb0\x20\xf7\x0e\xad\x15\x81\x56\xfc\x5b\x60\x99\xe1\x98\x7f\x72\x2d\xd3\x0b\x36\x33\x0f\x8d\xef\xa5\xc8\x1f\x94\xf2\xd1\x62\x73\x8b\x90\x46\x04\x4d\xb6\xb7\x5f\xcd\x18\x77\x60\xad\x03\xe3\xad\x3d\x68\xa9\xa2\x3d\xb9\x73\x7c\xcd\xd6\x03\x23\x75\x77\x4a\x79\x1f\x77\x35\x34\xc5\x16\xaa\xd3\xe4\x43\xf8\x90\xa4\xe5\x1d\x39\x2d\xb2\x84\xcb\xc0\x1d\x4e\xc0\x20\xe6\xe6\x80\x2c\xa6\xba\x23\x8b\xa9\xae\x49\x04\x84\x9a\xa4\x81\xba\x9d\x40\xbc\xb7\x68\x1c\xbb\xa4\x3a\x39\x25\x45\x04\x2b\xa0\x98\x19\x08\x12\x8a\x60\x27\x4f\x4a\x2b\xa1\x92\x7c\xd2\x30\xb4\x9b\x67\x5d\x6a\xc0\x00\x0f\x4e\x40\x9e\x82\xca\x70\xf2\x8f\x4e\x30\xfd\x96\xd1\xdc\xb5\x0e\x9b\x9f\x6c\x85\xa8\xc6\xae\xa3\xed\x8f\x2e\x1b\x5d\x14\x55\x7d\x6c\xfd\xd5\xc2\xeb\xa4\x48\x39\xc3\xdf\x07\xf1\x98\x4c\xde\xd8\xb9\x92\x0e\x4d\x0b\x83\x1f\xfa\xc5\x82\x29\x18\xf6\x6e\xa5\x7a\xdb\x31\x44\x53\xf6\xfb\xa8\x55\x2c\x48\x35\xe6\xb7\xdc\x44\x44\x66\x42\xb4\x0f\xea\x01\xf4\xc1\xe0\x29\xb0\xc0\x0c\x34\x4c\x5c\xdc\x69\x61\xdb\x09\x17\xa3\xb6\x07\xd2\x7f\xfe\x3b\xea\x3f\x02\xc3\x0c\xf6\x96\x42\xaa\xe4\xdb\xf2\x57\x97\xc0\x87\x99\xeb\x99\x5c\xb5\x5c\xf0\x42\x74\x7f\x84\x82\xd6\x34\x7f\x5b\xf2\xf9\xea\x0c\x2a\x06\x77\x14\xee\xa8\xa4\x09\x98\x49\x81\xf5\x18\xc3\x6a\xec\x7e\x80\x5b\x75\xb2\xe5\x49\x04\x68\x77\xfe\x5a\xb4\xab\x94\x5a\xdc\xdf\x3e\xcf\x16\x3f\x78\xf6\x75\x7d\x02\x5b\xeb\x39\xd8\x5d\xe6\xf6\x4d\x58\x97\x75\x5d\xa2\x4b\x7c\x93\x70\x4b\x4d\x28\xf8\x32\x3c\xc6\xbe\xce\xe3\x6a\x01\x09\xdc\x6e\x17\x60\x18\x48\xa4\x09\x4c\xae\x6b\xdb\xfc\x13\x30\xb9\x9a\x83\xbf\x01\x14\xa4\x98\xd9\xb1\x56\x71\x4d\xdb\xab\x11\xc1\x57\x24\xa7\xff\x27\x9c\x52\x1b\xbe\xe5\x02\x12\x88\x5b\x45\xb6\x51\x66\xaf\xde\x9e\x5e\x56\x87\x67\x93\x2d\x41\x00\x86\x45\xc9\xb5\xfe\xbb\x91\x80\xe0\x1f\xb4\x36\xa9\x45\x21\x6f\x94\xa6\x89\x57\x1a\x91\x8f\xbc\x48\x76\x5c\x49\x2a\x3a\x24\xc4\xcf\xf7\x69\x64\x80\x71\x76\x80\x14\x6f\x09\x2f\xd5\xe9\xc4\x07\xb4\xf5\xb3\x6a\x77\x96\x86\x18\x75\x0c\xdb\x8e\x4c\x0c\xdb\x01\x3d\x36\xef\x6a\xb1\xf9\x20\xb2\x1e\x16\x77\x94\xae\x57\xc6\x1b\x38\x43\x17\x98\xd0\x2d\xe6\xf3\xec\x79\xb4\xbf\xf4\x60\x55\x43\x8a\xb8\xd4\x8d\x54\xdd\x8e\xb6\x03\xc9\xb8\xb7\xee\xf5\xb6\x15\xcd\x59\xbd\xd9\xf9\xb5\xda\xd1\x9e\x0e\xc2\xc7\x4d\x71\xf8\x77\xbb\x36\xd1\xf6\x7a\xfc\xe3\x85\xbd\x44\x7b\xad\x77\xef\x23\x8e\x46\x60\xdd\xd9\x87\xba\xd5\x22\x12\xf6\x6b\xd4\x1f\xcb\xed\xc1\x94\xef\xf4\x0a\x57\x1c\x6a\x1b\xd6\xca\x16\x9a\xeb\x39\x75\x38\xa3\x0d\x9e\x8d\xe2\x09\xf5\xad\x3f\x0d\xca\xe4\xf7\xe6\x53\xf5\x0a\x78\xd0\x0a\x97\x00\x79\xae\xb8\x55\x78\xb6\xfe\x03\x76\xbc\xc8\x1c\x43\x09\xfe\x34\x34\xbd\x69\xfd\x3c\x2c\xcd\x1c\x7f\x9a\xda\x56\x66\x0f\x0e\xe6\xee\x13\xfb\x40\x2f\xd8\xee\x01\x4f\xd8\xc4\x2b\xc4\xd3\x81\x0c\x3b\xa0\xbc\x59\x81\x4e\xd3\xd2\xac\x50\x83\xcb\x37\xcb\xd5\x89\x87\xa2\x2c\x67\xed\x9e\xb8\xf9\x6e\x3e\xdc\x5e\x2f\x01\xe9\xee\x85\xed\xe3\xec\xb6\x20\x4b\x6f\xd6\x22\xd7\xfc\x52\x35\x7d\xc5\x0f\x01\x1e\xca\x37\x09\x11\xf1\xcf\xd3\xfd\xcb\xb2\x78\x26\x57\xea\x5e\xfa\xc4\xb8\x4b\x99\x45\xd6\xf9\xc2\x57\xc0\x72\x26\xe0\x4e\xc5\xd5\xef\x0a\x16\xbb\x9f\xc5\xf8\xaf\x53\xbd\x86\x14\x0c\x15\xa3\xff\x05\x00\x00\xff\xff\x56\x33\x4f\x09\x22\x28\x00\x00") +var _deployKubernetes117DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\xa6\xf7\x70\x07\x54\x56\x7c\x45\x81\x83\x00\x3f\xe4\x36\xbe\xad\xd1\x4d\x62\x24\xb9\x7b\x0d\x18\x6a\x2c\xb3\xa6\x48\x1e\x39\xf2\x46\x57\xf4\x7f\x2f\x48\x7d\x58\x5f\x56\x6c\x37\xd9\xa0\xf5\x3e\xac\x22\x71\x38\xbf\xf9\xe0\xcc\x6f\x2c\xff\x99\x7c\x06\x09\x86\x22\xc4\xe4\x2b\xc7\x0d\xb9\x48\xe9\x16\xc8\x36\xb3\xa8\x52\xfe\x07\x5c\xfc\x85\xc4\x8a\x48\x85\x04\x62\x8e\x7f\x9a\x4c\xa8\xe6\xbf\x81\xb1\x5c\xc9\x88\xec\x66\x93\x2d\x97\x71\x44\x1e\xc0\xec\x38\x83\x2b\xc6\x54\x26\x71\x92\x02\xd2\x98\x22\x8d\x26\x84\x08\xfa\x0c\xc2\xba\x2b\x42\x74\x0a\x69\xc0\x2c\x9f\x72\x89\x20\xa6\x4c\xa5\x61\x0c\x5a\xa8\x3c\x05\x89\x11\x89\xb9\x01\x86\x81\x36\x2a\xce\x18\x72\x25\x27\x84\x48\x9a\x42\x54\x0b\x06\x4c\x49\x34\x4a\x08\x30\xe5\x33\xab\x29\x83\x88\xc4\xb0\xa6\x99\xc0\x49\x10\x04\x2d\x84\xe6\x99\xb2\x29\xcd\x70\xa3\x0c\xff\x83\xba\x3d\xa7\xdb\x9f\xec\x94\xab\xb0\xc6\x7e\xaf\x04\xbc\x23\x62\x78\x41\x30\x92\x0a\xb7\x68\xc7\x1d\x2a\x30\x01\x5b\x27\xc3\xf8\x4d\x26\xc0\x46\x93\x80\x50\xcd\x3f\x1b\x95\x69\x8f\x23\x20\x17\x17\x13\x42\x0c\x58\x95\x19\x06\xe5\x3d\x90\xb1\x56\x5c\xa2\x9d\x10\xb2\x03\xf3\x5c\xde\x4e\x00\xfd\xff\x5f\x29\xb2\x8d\xbf\x12\xdc\x16\xb7\x62\x10\x80\xe0\x2f\x33\x1d\xd3\xf2\x92\x19\x70\x97\x3d\x9d\x4c\x29\x13\x73\xd9\xf4\x5a\x1f\x84\x00\x6a\xe1\xbd\x10\x58\x54\x86\x26\x70\x50\x39\xb3\xbc\x5c\xc2\xa8\xa6\x8c\x23\x3f\x04\xa5\x06\xb0\xc7\x54\x2a\xed\x40\xd1\xf5\xf3\x12\xea\x51\xa1\xd0\x2a\x1e\x50\xdc\x13\xa5\x5a\xdb\xbe\xb0\x01\x2d\x38\xa3\x16\x86\x22\x79\x46\x42\x7f\x12\x99\x45\x30\x1f\x91\xd7\x26\x93\x12\xcc\x49\x59\xac\x9d\x65\x16\x41\xe2\x4e\x89\x2c\x3d\x2f\x80\x27\xc5\xaa\xa3\x90\x09\xca\xd3\xe3\xb5\x96\xb9\x72\x72\xae\x56\x89\x2a\xa8\x3d\x78\x60\x3a\xea\x8e\xab\x02\x3b\xe8\x95\x80\x13\xb3\xbd\x6f\x8b\xa4\xda\x6e\x14\x4e\x5f\x33\xaa\x0c\x59\xb9\x7c\xcc\xaa\xb7\xd2\xe1\x1a\x40\xdf\xe0\x57\x54\xbd\x5e\x46\xa4\x8a\xdf\x32\x28\xff\xc5\x76\xc7\xb9\x83\x22\x52\xb6\x49\x5f\xf1\x44\x43\xcd\x79\x7d\xf1\x67\x2e\x63\x2e\x93\xf7\x6c\xe8\x96\xb7\x2b\x88\x12\x30\xd2\x1e\x95\x80\x7b\x58\x3b\xe5\x95\xdb\x46\x6c\x99\x10\xd2\x68\xf1\xc7\x77\x66\x9b\x3d\xff\x13\x18\xfa\x0a\x36\xc8\x6f\xbe\x05\x2d\x69\x54\xf1\x8f\x89\xc2\xb9\xce\x6e\xb6\x9f\x93\xba\xc6\xfb\xbb\xbd\xcb\x57\x3f\x90\xa8\x5a\x0d\xcc\xe9\xd2\xca\x60\x4d\x22\x0c\x46\x64\x76\x79\x79\x79\xe9\x41\x20\x35\x09\xe0\xaa\x75\xd7\x82\x00\x86\xca\x14\x30\xa9\xd6\x87\x54\x9e\x61\xc3\x07\x7a\x2b\x05\x34\x9c\xd9\xd3\x5d\x35\x1b\x74\xd5\xec\xbd\x5d\x45\x08\xe6\x1a\x22\x72\xab\x62\x70\x5a\x7b\xbe\x73\x4c\x6f\x7f\x96\x1f\x90\x22\xac\x33\xf1\x00\x1f\x39\x1b\x55\x7e\xac\xf8\x66\x44\x66\x3d\x37\xa5\xae\x5b\x7c\x69\xc0\x7a\xc5\x73\x67\xe1\xb6\x45\x3e\xdd\x8e\xc0\x47\x48\xb5\xa0\x08\x25\xaa\x86\xcf\x3c\x24\x29\x15\xfa\xe2\x53\xa3\x1c\x04\x62\x99\xa1\x2e\x4c\x6e\x6b\xca\x25\x18\x5b\xae\x16\x2d\x0b\x5f\xb5\xf1\x2c\x2b\x0f\x0a\x7e\x85\xe7\x8d\x52\xdb\x88\xf0\x44\x2a\x03\x7e\x69\x15\x1b\x0f\x66\xbd\xe6\x92\x63\xbe\x87\xe7\xe8\xc4\x55\xef\xae\x8b\xe4\xef\x19\x37\x10\x5f\x67\x86\xcb\xe4\x81\x6d\x20\xce\x04\x97\xc9\xd2\x6f\x5c\xde\x5e\xbc\x00\xcb\x1c\xa6\xa6\x64\xb1\xe7\x43\x19\xfb\x47\x30\xa9\x6d\x3f\x0e\x8a\x54\x58\xbc\x68\x03\xd6\xb6\x3d\x5d\xad\xd8\x42\x1e\x0d\x19\x38\xe0\xbf\xea\xa3\x34\x18\xea\xb2\x8d\xdc\x2a\x5c\xca\xde\xf3\x1d\x15\x19\xf4\x54\x79\x9a\x25\xd5\xc5\xd0\xed\x35\x15\x16\xaa\x27\xfb\x40\x57\x5b\xb8\x31\x36\x4d\xa9\x8c\xf7\x7b\x06\x24\xcc\xac\x09\x85\x62\x54\x84\xcf\x5c\x86\x75\xe0\x63\xc3\x77\x0d\xd0\x01\x09\x76\xf3\xbf\x36\xff\x4c\x55\x0c\xf3\x01\xf3\x02\x12\x54\xd3\xf8\x3c\x93\xfc\x25\x0a\xc3\x90\x59\x1e\xb6\xb3\x69\x6a\x15\xdb\x36\x65\x0c\x24\xdc\xa2\xc9\x17\x95\x2c\x32\x1d\x85\xe1\xe5\xd4\xff\x8b\xf6\xad\xa0\x5c\xef\x62\xc6\xe3\xf9\x77\xdf\xff\xe3\xd7\x9f\x17\x4f\xb7\x77\xd7\x8b\xa7\xdb\xab\x9b\xc5\x0f\xcd\x35\x8c\xfe\xc2\x05\xcc\x43\x06\x06\x6d\xc8\xe8\x94\x19\x6c\x3d\x07\x83\xcd\x15\x28\x6c\x77\xc9\x16\xf2\xee\x8a\x2d\xe4\xcd\x15\x85\xa3\x5c\x8d\x99\x7f\xf7\xfd\xea\x66\x71\xf3\xf4\xe9\x61\xf9\x74\x7d\xbf\xfc\x6d\x71\xdf\x87\x54\x56\xf7\x2f\x7e\xea\x9a\x47\xfb\xb2\xed\x3e\x20\x77\xcd\xd0\x14\x35\xad\x6d\xdf\xa4\x93\x1f\xbf\x18\x95\xb6\x53\x64\xcd\x41\xc4\x25\x57\x69\x7e\x3a\xed\xac\xfd\xd0\x0b\xad\x28\x6e\x22\x7f\xfc\xa6\xce\xbb\xae\x26\xf5\xd0\x3c\x2e\xee\x6f\x96\xb7\x57\x8f\xcb\xbb\xdb\xa7\x2f\x77\x9f\x9f\x56\x57\x8f\x7f\xef\x62\x8a\x48\x88\xa9\x0e\x11\x4c\x5a\x7e\x65\x12\x08\x95\xf4\xb6\x1a\xf2\x55\x7f\xab\xfe\x99\xaa\xd7\xf0\x94\x26\x10\x11\xff\xa0\x9b\xb8\x11\xa3\x92\x9a\xbc\xbd\x76\x95\x09\xb1\x52\x82\xb3\x3c\x22\xcb\xf5\xad\xc2\x95\x01\x0b\x72\x1f\xf0\x46\x17\xe9\xe4\x7f\xdd\x70\x2b\x1b\xea\xe3\xb5\xea\xf4\xdf\xe6\x4e\xfb\x5e\x5e\x7c\x2c\xb0\xcc\x70\xcc\x3f\xb9\xc9\xed\x05\xdb\xf5\x8b\xc6\x77\x52\xe4\xf7\x4a\xf9\x9c\xb4\xb9\x45\x48\x23\x82\x26\xdb\x47\xa1\xe1\xd2\x1b\xb0\xd6\x99\xe4\x63\x36\xea\xef\x62\x4a\xba\x71\xb4\xb1\x65\x41\xea\xee\x94\xf2\x3e\xbb\x7b\xf0\xab\x33\xe9\x0f\xca\x21\x49\xcb\x7b\x72\x5a\x64\x09\x97\x81\x3b\xe2\x80\x41\xcc\xcd\x01\x59\x4c\x75\x4f\x16\x53\xdd\x90\x08\x08\x35\x49\x0b\x75\xb7\x0c\xf9\x98\xd3\x38\x76\xa5\x79\x7e\x4c\xa1\x09\xd6\x40\x31\x33\x10\x24\x14\xc1\xce\x1f\x95\x56\x42\x25\xf9\xbc\xe5\x68\xb7\xce\xba\xd8\x61\x80\x07\x17\x20\x4f\x41\x65\x38\xff\x5b\xda\xba\x5d\x06\xbd\x06\xe5\x0f\xf9\xac\x9b\xb5\xbf\x67\x34\x77\x43\xce\xf6\x27\x5b\x81\x6e\xcc\x01\xd1\xee\x47\x57\xf6\xce\x4a\xdf\xa1\xb9\xe2\xa4\x3c\x9e\xbd\x7b\x1e\x1f\x95\x92\x27\x24\xd6\x41\x3c\x26\x93\x57\xf6\x56\x49\x87\xa6\x83\xc1\x3f\xfa\xd5\x82\x29\x26\x8a\x7a\xa7\xe6\x98\x35\x46\xcb\xf6\x76\x34\x1a\x2c\xa4\x1a\xf3\x6b\x6e\x22\x22\x33\x21\xba\x75\xe5\x00\xfa\x60\xf4\xb8\x59\x60\x06\x5a\x2e\x2e\xee\x74\xb0\xd5\xc2\xc5\x53\x3b\x00\xe9\x5f\xff\xee\x00\xaa\xce\xda\x38\x63\xbf\xa6\x90\x2a\xf9\xbe\x7c\xdd\xf5\x9b\x71\xa6\x7e\x22\x37\x2f\x37\x3c\x13\xdd\x87\x52\xee\x06\xf4\x6f\x4b\xb6\xdf\x9c\x31\xc6\xe0\xce\xd2\x0d\x95\x34\x01\x33\x2f\xb0\xbe\xc6\x28\x5b\xd6\x8f\x70\xc9\x5e\x5d\x3f\x8a\x10\xd6\x07\xb8\x43\x33\x4b\xa9\xd5\xdd\xf5\xd3\x72\xf5\x83\x67\x9b\xb3\x23\xd8\xe9\x40\x65\xe8\x33\xd5\x6f\xc2\x42\xad\x1b\xab\x5d\xe5\x9c\x87\x3b\x6a\x42\xc1\x9f\xc3\x23\xd8\xe8\x69\xdc\xd5\x59\xbb\x02\xc3\x40\x22\x4d\x60\x3e\x6b\x9b\xf9\xbf\xce\x6c\x1b\x09\xf0\x0d\xa0\x20\xc5\xcc\x4e\xb5\x8a\x1b\xda\xde\x8c\x18\xbf\x21\x59\xff\xbf\xe2\xd8\xda\xf0\x1d\x17\x90\x40\xdc\xe1\x02\x2d\x36\x70\xf9\xfe\x74\xbb\x3a\xa2\xdb\xec\x19\x04\x60\x58\x30\x03\xeb\xff\x36\x12\x10\xfc\xf7\xdf\x6d\x06\x54\xc8\x1b\xa5\x69\xe2\x95\x46\xe4\x67\x5e\x94\x54\xae\x24\x15\x7d\xbf\xb8\xf5\xbe\x58\x8d\x30\xf0\x1e\x90\xe2\xe5\xed\xb9\x3a\x9d\xf8\x88\xb6\xe1\x29\xc3\x9d\xc8\xb1\x09\x23\x86\x5d\x4f\x26\x86\xdd\x88\x1e\x9b\xf7\xb5\xd8\x7c\x14\xd9\x00\xd9\x7c\x75\x7c\xa9\x9c\x37\x72\x12\xcf\x70\xa1\xdb\xcc\x57\xf3\xd3\xc6\xa0\x32\x82\x55\xa7\x2a\xf2\x52\xb7\x1a\x42\x37\xdb\x0e\x94\xfc\xc1\xee\x3a\x38\x66\xb5\x57\x0d\xd6\xf8\xb7\x1a\xf2\x07\xc6\x25\x9f\x37\x45\x09\xa9\xad\x36\xd1\x6e\x36\xfd\xf1\xcc\xc1\xa9\xbb\xd7\x87\x8f\x3b\xaf\x66\x60\x33\xd8\x87\xa6\xf7\x22\x13\xf6\x7b\x34\xbf\xec\xdc\x83\x29\x5f\xb5\x16\xa1\x38\x34\xdd\x6c\x94\x2d\x34\x37\x6b\xea\x78\x45\x1b\x3d\x1b\xc5\x8b\x83\x6b\x7f\x1a\x94\xc9\xef\xcc\xa7\xea\xcd\xfc\xa8\x17\xce\x01\xf2\x54\x31\xb8\xf0\x64\xfd\x07\xfc\x78\x96\x3b\xc6\x0a\xfc\x71\x68\x06\xcb\xfa\x69\x58\xda\x35\xfe\x38\xb5\x9d\xca\x1e\x1c\xac\xdd\x47\x8e\xab\x5e\xb0\x3b\xaa\x1e\x61\xc4\x1b\xe4\xd3\x81\x0a\x3b\xa2\xbc\xdd\x81\x8e\xd3\xd2\xee\x50\xa3\xdb\xb7\xdb\xd5\x91\x87\xa2\x6c\x67\xdd\xd1\xbd\xfd\x93\x89\x70\x37\x7b\x06\xa4\xf5\x7b\xf4\x87\xe5\x75\x41\xb9\xde\x6d\x92\x6f\xc4\xa5\x1a\x2d\x8b\xdf\x67\xdc\x97\xef\x67\x22\xe2\xdf\x52\xf8\x77\x98\xf1\x52\xae\xd5\x9d\xf4\x85\xb1\x2e\x99\x45\xd5\xf9\xc2\xd7\xc0\x72\x26\xe0\x46\xc5\xd5\xcf\x3d\x56\xf5\xaf\x95\xfc\x9f\x0b\xbd\x81\x14\x0c\x15\x93\xff\x04\x00\x00\xff\xff\x27\xe4\x4c\xed\xb9\x29\x00\x00") func deployKubernetes117DirectPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -99,12 +99,12 @@ func deployKubernetes117DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.17/direct/pmem-csi.yaml", size: 10274, mode: os.FileMode(420), modTime: time.Unix(1599471423, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.17/direct/pmem-csi.yaml", size: 10681, mode: os.FileMode(420), modTime: time.Unix(1599822735, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes117LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xd3\x7d\xd8\x05\x2a\x2b\xde\xa2\xc0\x42\x80\x1f\x32\x13\xef\xd4\xe8\xc4\x31\x92\xec\xbe\x06\x34\x75\x2c\xb3\xa6\x48\x2e\x79\xa4\x89\xb6\xe8\x7f\x2f\xa8\x8b\xad\x9b\xe5\xcb\xe6\xd2\xcd\x3c\x8c\x23\xf2\xf0\x7c\x3c\xd7\xef\x58\xf9\x2b\xf9\x02\x12\x0c\x45\x08\xc9\x37\x8e\x1b\x72\x15\xd3\x2d\x90\x6d\x62\x51\xc5\xfc\x77\xb8\xfa\x1b\x09\x15\x91\x0a\x09\x84\x1c\xff\x32\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x01\x49\x27\xa3\x2d\x97\x61\x40\x1e\xc1\xa4\x9c\xc1\x0d\x63\x2a\x91\x38\x8a\x01\x69\x48\x91\x06\x23\x42\x04\x5d\x81\xb0\xee\x13\x21\x3a\x86\xd8\x63\x96\x8f\xb9\x44\x10\x63\xa6\x62\x3f\x04\x2d\x54\x16\x83\xc4\x80\x88\x34\xf6\xb4\x51\x61\xc2\x90\x2b\x39\x22\x44\xd2\x18\x82\x9d\x94\xc7\x94\x44\xa3\x84\x00\x53\xae\x59\x4d\x19\x04\x24\x84\x35\x4d\x04\x8e\x3c\xcf\x6b\xc0\x33\x2b\xca\xc6\x34\xc1\x8d\x32\xfc\x77\xea\xce\x1c\x6f\x7f\xb2\x63\xae\xfc\x1d\xf0\x07\x25\xe0\xad\xe0\xc2\x0b\x82\x91\x54\xb8\x4d\x29\x77\x90\xc0\x78\x6c\x1d\xf5\x83\x37\x89\x00\x1b\x8c\x3c\x42\x35\xff\x62\x54\xa2\x73\x10\x1e\xb9\xba\x1a\x11\x62\xc0\xaa\xc4\x30\x28\x9f\x81\x0c\xb5\xe2\x12\xed\x88\x90\x14\xcc\xaa\x7c\x1c\x01\xe6\xff\x7f\xa3\xc8\x36\xf9\x27\xc1\x6d\xf1\x28\x04\x01\x08\xf9\xc7\x44\x87\xb4\xfc\xc8\x0c\xb8\x8f\x1d\x9d\x4c\x29\x13\x72\x59\x37\x59\x17\x84\x00\x6a\xe1\xad\x10\x58\x54\x86\x46\x70\x50\x39\xb3\xbc\xdc\xc2\xa8\xa6\x8c\x23\x3f\x04\x65\x07\x60\x8f\xa9\x54\xda\x82\xa2\x77\xeb\x25\xd4\x93\x5c\xa1\x55\xd8\xa3\xb8\x23\x4a\xb5\xb6\x5d\x61\x03\x5a\x70\x46\x2d\xf4\x79\xf2\x82\x68\xfe\x2c\x12\x8b\x60\xde\x3d\xa8\x4d\x22\x25\x98\xb3\x42\x58\xbb\x6b\x59\x04\x89\xa9\x12\x49\x7c\x99\xf7\xce\x72\x54\x4b\x21\x13\x94\xc7\xa7\x6b\x2d\x03\xe5\xec\x40\xad\xa2\x54\x50\x7b\x30\x5b\x5a\xea\x4e\x2b\x01\x29\x74\xf2\xff\xcc\x50\xef\xde\x45\x52\x6d\x37\x0a\xc7\xc7\x2e\x55\xba\xac\xdc\x3e\x74\xab\xd7\xd2\xe1\x4a\x7f\xf7\xc2\x47\x54\x1d\xaf\x21\x52\x85\xaf\xe9\x94\x3f\x70\xdc\x69\xe6\xa0\x88\x94\x6d\xe2\x23\x96\xa8\xa9\xb9\xac\x23\x7e\xe2\x32\xe4\x32\x7a\xb3\x3e\x6e\x79\xb3\x7c\x28\x01\x03\x8d\x51\x09\x78\x80\xb5\xd3\x5c\xd9\x6c\xe0\x22\x23\x42\x6a\x9d\xfd\xf4\x9e\x6c\x93\xd5\xbf\x81\x61\x5e\xbe\x7a\x39\xcd\x7b\xb0\x91\x5a\xfd\xfe\x00\x17\x5c\x6a\xe9\x7a\xd7\x39\xab\x5f\xbc\xbd\xcd\xdb\x04\xf5\xa3\x98\xa9\xd5\xc0\x9c\x22\xad\x0c\xee\x88\x83\xc1\x80\x4c\xae\xaf\xaf\xaf\x73\x04\x48\x4d\x04\xb8\x6c\x3c\xb5\x20\x80\xa1\x32\x05\x46\xaa\xf5\x21\x95\xe7\x5e\xe0\xa3\xec\x14\x03\x1a\xce\xec\xf9\x46\x9a\xf4\x1a\x69\xf2\xa6\x46\x22\x04\x33\x0d\x01\x59\xa8\x10\x9c\xca\x8e\xd5\x1c\xa9\xdb\x27\xef\x23\x52\x84\x75\x22\x1e\xe1\xc3\x06\xa0\xca\x82\x15\xaf\x0c\xc8\xa4\x63\xa0\xd8\x35\x86\xaf\x35\x4c\x47\x6c\x76\x3e\x68\x5b\xc4\xd0\x62\x00\x3b\x42\xac\x05\x45\x28\x21\xd5\xac\xe5\x7e\x44\x03\xdd\x51\x7c\xe7\x23\x3c\x28\xf5\x0d\x56\x1b\xa5\xb6\x01\xe1\x91\x54\x06\xf2\xad\x95\x51\x73\x24\xeb\x35\x97\x1c\xb3\x3d\x36\xd7\xf2\x6f\x3a\x4f\x9d\x0b\x7e\x4b\xb8\x81\xf0\x36\x31\x5c\x46\x8f\x6c\x03\x61\x22\xb8\x8c\xe6\xf9\xc1\xe5\xe3\xd9\x0b\xb0\xc4\x61\xaa\x4b\x16\x67\x3e\x96\x4e\x7b\x02\x13\xdb\xe6\xb2\x57\xf8\x70\xf6\xa2\x0d\x58\x17\x89\xad\x75\xb7\x63\x0b\x59\xd0\x77\xc1\x1e\xe3\x55\x3f\x4a\x83\xa1\x2e\x4c\xc8\x42\xe1\x5c\x76\xd6\x53\x2a\x12\xe8\xa8\xca\xa9\x90\x54\x57\x7d\x8f\xd7\x54\x58\xa8\x56\x9c\x66\xca\x25\x98\xdd\x11\x6e\xce\x8c\x63\x2a\xc3\xfd\x99\x1e\xf1\x13\x6b\x7c\xa1\x18\x15\xfe\x8a\x4b\x7f\xe7\xf5\xd0\xf0\xb4\x06\xda\x23\x5e\x3a\xfd\x7b\xfd\xd7\x58\x85\x30\xed\xb9\x9e\x47\xbc\x6a\x5c\x9e\x26\x92\xbf\x04\xbe\xef\x33\xcb\xfd\x66\x28\x8d\xad\x62\xdb\xba\x8c\x81\x88\x5b\x34\xd9\xac\x92\x45\xa6\x03\xdf\xbf\x1e\xe7\xff\x82\x7d\xdd\xae\xd4\x17\x95\xed\x6b\x3e\x62\x4c\x83\x7d\xc9\x2a\xd7\x9d\x4f\x79\x38\xfd\xee\xfb\x7f\xfd\xf2\x69\xf6\xbc\xb8\xbf\x9d\x3d\x2f\x6e\xee\x66\x3f\xd4\xf7\x30\xfa\x33\x17\x30\xf5\x19\x18\xb4\x3e\xa3\x63\x66\xb0\xb1\x0e\x06\xeb\x3b\x50\xd8\xf6\x96\x2d\x64\xed\x1d\x5b\xc8\xea\x3b\x0a\x43\xba\xe2\x31\xfd\xee\xfb\xe5\xdd\xec\xee\xf9\xf3\xe3\xfc\xf9\xf6\x61\xfe\xeb\xec\xa1\x05\x09\x64\x5a\x77\x4d\x51\x8c\x9a\xf8\x47\xad\xf8\xf8\xd9\xa8\xb8\x19\x22\x6b\x0e\x22\x2c\x59\x45\xfd\xa7\xd5\x7e\x9a\x8b\xb9\xd0\x92\xe2\x26\xc8\xd3\x6f\xec\xac\xe7\xea\x49\x07\xcd\xd3\xec\xe1\x6e\xbe\xb8\x79\x9a\xdf\x2f\x9e\xbf\xde\x7f\x79\x5e\xde\x3c\xfd\xb3\x8d\x29\x20\x3e\xc6\xda\x47\x30\x71\xf9\x9d\x86\x27\x54\xd4\x39\xaa\xcf\x16\xdd\xa3\xba\x39\xb5\xdb\xc3\x63\x1a\x41\x40\xf2\x85\x76\xe0\x06\x8c\x4a\x6a\xb2\xe6\xde\x65\x22\xc4\x52\x09\xce\xb2\x80\xcc\xd7\x0b\x85\x4b\x03\x16\xe4\xde\xa1\xb5\xf2\xdf\x8a\x7f\x0b\x2c\x31\x1c\xb3\xcf\x6e\x2e\x7a\xc1\x66\xe5\xa1\xe1\xbd\x14\xd9\x83\x52\x79\xb4\xd8\xcc\x22\xc4\x01\x41\x93\xec\xed\x57\x33\xc6\x1d\x58\xeb\xc0\xe4\xd6\x1e\xb4\x54\x31\x83\xdc\x39\x6a\x66\xeb\x81\x11\xbb\x27\xa5\x7c\x1e\x77\x35\x34\xc5\x15\xaa\x6c\xca\x43\xf8\x90\xa4\xe5\x1d\x39\x2d\x92\x88\x4b\xcf\x25\x27\xa0\x17\x72\x73\x40\x16\x63\xdd\x91\xc5\x58\xd7\x24\x3c\x42\x4d\xd4\x40\xdd\x2e\x20\xb9\xb7\x68\x18\xba\xa2\x3a\x3d\xa5\x44\x78\x6b\xa0\x98\x18\xf0\x22\x8a\x60\xa7\x4f\x4a\x2b\xa1\xa2\x6c\xda\x30\xb4\xdb\x67\x5d\x69\x40\x0f\x0f\x6e\x40\x1e\x83\x4a\x70\xfa\x8f\x4e\x30\xfd\x96\xd0\xcc\x8d\x08\xdb\x9f\x6c\x85\xa8\x46\xa4\x83\xf4\x47\x57\x8d\x2e\x8a\xaa\x3e\x62\xfe\x6a\xe1\x75\x52\xa4\x9c\xe1\xef\x83\x78\x4c\x22\x6f\xec\x42\x49\x87\xa6\x85\x21\x5f\xfa\xc5\x82\x29\xf8\xf4\xee\xa4\xfa\x84\x31\xc4\x51\xf6\xf7\xa8\x75\x2c\x88\x35\x66\xb7\xdc\x04\x44\x26\x42\xb4\x13\xf5\x00\x7a\x6f\x30\x0b\x2c\x30\x03\x0d\x13\x17\x4f\x5a\xd8\x76\xc2\xc5\xaa\xed\x81\xf4\x9f\xff\x8e\xfa\x53\x60\x98\xb8\xde\x52\x88\x95\x7c\x43\xda\xea\xaa\xf7\x30\x61\x3d\x93\xa2\x96\x07\x5e\x02\xed\x8f\x30\xcf\x9a\xda\x77\xe4\x9c\xaf\x4e\x9c\x42\x70\x19\x70\x47\x25\x8d\xc0\x4c\x45\x1a\x1f\x63\x55\x8d\x7b\x0f\xf0\xa9\x4e\x85\x3c\x89\xf4\xec\x72\xae\x45\xb5\x4a\xa9\xe5\xfd\xed\xf3\x7c\xf9\x43\xce\xb8\x26\x27\x30\xb4\x9e\x64\xee\xb2\xb5\x77\x61\x5a\xd6\x0d\x84\xae\xd8\x4d\xfd\x94\x1a\x5f\xf0\x95\x7f\x8c\x71\x9d\xc7\xcf\x3c\xe2\xb9\xdb\x2e\xc1\x30\x90\x48\x23\x98\x4e\x6a\xd7\xfc\x13\xb0\xb7\x9a\x83\xdf\x01\x0a\x52\x4c\xec\x58\xab\xb0\xa6\xed\xd5\xc8\xdf\x2b\x12\xd2\xff\x13\x1e\xa9\x0d\x4f\xb9\x80\x08\xc2\x56\x63\x6d\xb4\xd6\xeb\xb7\xa7\x94\x55\xf2\x6c\x93\x15\x08\x40\xbf\x68\xb3\x36\xff\xdd\x48\x40\xc8\xbf\x44\x6d\xd2\x89\x42\xde\x28\x4d\xa3\x5c\x69\x40\x3e\xf1\x90\x1b\xc8\xab\x31\x15\x1d\xe2\x91\xef\xcf\xcb\xc8\x00\xcb\xec\x00\x29\xde\xfd\x5d\xaa\xd3\x89\x0f\x68\xeb\x67\xd2\x2e\x97\x86\x58\x74\x08\x69\x47\x26\x84\x74\x40\x8f\xcd\xba\x5a\x6c\x36\x88\xac\x87\xb9\x1d\xa5\xe8\x95\xf1\x06\x72\xe8\x02\x13\xba\xc3\xf2\x3a\x7b\x1e\xd5\x2f\x3d\x58\xf5\x90\x22\x2e\x75\xa3\x54\xb7\xa3\xed\x40\x31\xee\xed\x7b\xbd\xa3\x44\x73\x57\x6f\x75\x7e\xad\x11\xb4\x67\x6a\xc8\xe3\xa6\x48\xfe\xdd\xad\x4d\x90\x4e\xc6\x3f\x5e\x38\x3f\xb4\xcf\xfa\xf0\xd9\xe1\x68\x04\xd6\x9d\x7d\x68\x42\x2d\x22\x61\x7f\x46\xfd\xab\xb8\x3d\x98\xf2\x65\x5d\xe1\x8a\x43\xa3\xc2\x46\xd9\x42\x73\xbd\xa6\x0e\x57\xb4\xc1\xdc\x28\xbe\x8c\xbe\xcd\xb3\x41\x99\xec\xde\x7c\xae\xde\xed\x0e\x5a\xe1\x12\x20\xcf\x15\xb7\xf2\xcf\xd6\x7f\xc0\x8e\x17\x99\x63\xa8\xc0\x9f\x86\xa6\xb7\xac\x9f\x87\xa5\x59\xe3\x4f\x53\xdb\xaa\xec\xde\xc1\xda\x7d\xe2\xec\x97\x0b\xb6\xe7\xbe\x13\x2e\xf1\x0a\xf1\x74\xa0\xc2\x0e\x28\x6f\x76\xa0\xd3\xb4\x34\x3b\xd4\xe0\xf1\xcd\x76\x75\x62\x52\x94\xed\xac\x3d\x07\x37\x5f\xba\xfb\xe9\x64\x05\x48\x77\x2f\x63\x1f\xe7\xb7\x05\x59\x7a\x9b\xb1\xb8\xe6\x94\x6a\xdc\x2b\x5e\xef\x3f\x94\xaf\x0e\x02\x92\x7f\x81\x9e\xbf\x11\x0b\xe7\x72\xad\xee\x65\x5e\x15\x77\xf5\xb2\x28\x39\x5f\xf9\x1a\x58\xc6\x04\xdc\xa9\xb0\xfa\x6b\x81\xe5\xee\x8f\x5d\xf2\x5f\x67\x7a\x03\x31\x18\x2a\x46\xff\x0b\x00\x00\xff\xff\xf6\xb2\x6e\xe3\xef\x27\x00\x00") +var _deployKubernetes117LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xa7\xfb\xb0\x0b\x54\x56\xbc\x45\x81\x85\x00\x3f\x64\x27\xde\xa9\xd1\x49\x62\x24\xd9\x7d\x0d\x18\xea\x58\x66\x4d\x91\x5c\xf2\x48\x13\x6d\xd1\xff\x5e\x90\x92\x6c\xdd\xac\xd8\x6e\x2e\x68\x3d\x0f\xa3\x48\x24\xcf\x77\xee\xdf\xb1\xfc\x67\xf2\x05\x24\x18\x8a\x10\x93\x6f\x1c\x37\xe4\x53\x4a\xb7\x40\xb6\x99\x45\x95\xf2\x3f\xe0\xd3\x5f\x48\xac\x88\x54\x48\x20\xe6\xf8\xa7\xc9\x84\x6a\xfe\x1b\x18\xcb\x95\x8c\x48\x3e\x9b\x6c\xb9\x8c\x23\x72\x0f\x26\xe7\x0c\x2e\x19\x53\x99\xc4\x49\x0a\x48\x63\x8a\x34\x9a\x10\x22\xe8\x13\x08\xeb\xae\x08\xd1\x29\xa4\x01\xb3\x7c\xca\x25\x82\x98\x32\x95\x86\x31\x68\xa1\x8a\x14\x24\x46\x44\xe4\x69\xa0\x8d\x8a\x33\x86\x5c\xc9\x09\x21\x92\xa6\x10\xed\x76\x05\x4c\x49\x34\x4a\x08\x30\xd5\x33\xab\x29\x83\x88\xc4\xb0\xa6\x99\xc0\x49\x10\x04\x2d\x78\xe6\x89\xb2\x29\xcd\x70\xa3\x0c\xff\x83\xba\x33\xa7\xdb\x9f\xec\x94\xab\x70\x07\xfc\x4e\x09\x78\x2b\xb8\xf0\x8c\x60\x24\x15\x6e\x51\xce\x1d\x24\x30\x01\x5b\x27\xc3\xe0\x4d\x26\xc0\x46\x93\x80\x50\xcd\xbf\x18\x95\x69\x0f\x22\x20\x9f\x3e\x4d\x08\x31\x60\x55\x66\x18\x54\xf7\x40\xc6\x5a\x71\x89\x76\x42\x48\x0e\xe6\xa9\xba\x9d\x00\xfa\xff\xbf\x51\x64\x1b\x7f\x25\xb8\x2d\x6f\xc5\x20\x00\xc1\x5f\x66\x3a\xa6\xd5\x25\x33\xe0\x2e\x7b\x32\x99\x52\x26\xe6\xb2\x69\xb2\x3e\x08\x01\xd4\xc2\x5b\x21\xb0\xa8\x0c\x4d\xe0\xa0\x70\x66\x79\xb5\x84\x51\x4d\x19\x47\x7e\x08\xca\x0e\xc0\x1e\x53\x25\xb4\x03\x45\xef\x9e\x57\x50\x8f\x72\x85\x56\xf1\x80\xe0\xde\x56\xaa\xb5\xed\x6f\x36\xa0\x05\x67\xd4\xc2\x90\x27\xcf\x88\xe6\xcf\x22\xb3\x08\xe6\xdd\x83\xda\x64\x52\x82\x39\x29\x84\xb5\x53\xcb\x22\x48\xcc\x95\xc8\xd2\xf3\xbc\x77\x92\xa3\x3a\x02\x99\xa0\x3c\x3d\x5e\x6a\x15\x28\x27\x07\x6a\x1d\xa5\x82\xda\x83\xd9\xd2\x11\x77\x5c\x09\xc8\xa1\x97\xff\x27\x86\x7a\x5f\x17\x49\xb5\xdd\x28\x9c\xbe\xa4\x54\xe5\xb2\x6a\xf9\x98\x56\xaf\x25\xc3\x95\xfe\xbe\xc2\x2f\x88\x7a\xb9\x86\x48\x15\xbf\xa6\x53\xfe\x8b\xe3\x8e\x33\x07\x45\xa4\x6c\x93\xbe\x60\x89\x86\x98\xf3\x3a\xe2\xcf\x5c\xc6\x5c\x26\x6f\xd6\xc7\x2d\x6f\x97\x0f\x25\x60\xa4\x31\x2a\x01\x77\xb0\x76\x92\x6b\x9b\x8d\x28\x32\x21\xa4\xd1\xd9\x8f\xef\xc9\x36\x7b\xfa\x27\x30\xf4\xe5\x6b\x90\xd3\xbc\x07\x1b\x69\xd4\xef\x0f\x70\xc1\xb9\x96\x6e\x76\x9d\x93\xfa\xc5\xdb\xdb\xbc\x4b\x50\x3f\x8a\x99\x5a\x0d\xcc\x09\xd2\xca\xe0\x8e\x38\x18\x8c\xc8\xec\xe2\xe2\xe2\xc2\x23\x40\x6a\x12\xc0\x55\xeb\xae\x05\x01\x0c\x95\x29\x31\x52\xad\x0f\x89\x3c\x55\x81\x8f\xb2\x53\x0a\x68\x38\xb3\xa7\x1b\x69\x36\x68\xa4\xd9\x9b\x1a\x89\x10\x2c\x34\x44\xe4\x46\xc5\xe0\x44\xf6\xac\xe6\x48\xdd\x3e\x79\xef\x91\x22\xac\x33\x71\x0f\x1f\x36\x00\xd5\x16\xac\x79\x65\x44\x66\x3d\x03\xa5\xae\x31\x7c\x6d\x60\x7a\xc1\x66\xa7\x83\xb6\x65\x0c\xdd\x8c\x60\x47\x48\xb5\xa0\x08\x15\xa4\x86\xb5\x3c\x1e\x29\x15\xfa\x52\xb3\x83\x38\x88\xc2\x32\x43\x9d\x83\xdc\xd1\x94\x4b\x30\xb6\x5a\x2d\x5a\xea\xbd\xa8\xe0\xe9\x2a\x1e\xdc\xf5\x0d\x9e\x36\x4a\x6d\x23\xc2\x13\xa9\x0c\xf8\xa5\xb5\x57\x3c\x92\xf5\x9a\x4b\x8e\xc5\x1e\x9b\xe3\x0c\x97\xbd\xbb\xce\x87\xbf\x67\xdc\x40\x7c\x95\x19\x2e\x93\x7b\xb6\x81\x38\x13\x5c\x26\x4b\x7f\x70\x75\x7b\xf1\x0c\x2c\x73\x98\x9a\x3b\xcb\x33\xef\x2b\xaf\x3f\x80\x49\x6d\xfb\x71\x50\x06\xc1\xe2\x59\x1b\xb0\xb6\x6d\xe6\x7a\xc5\x16\x8a\x68\x48\xc1\x01\xe3\xd5\x1f\xa5\xc1\x50\x17\x67\xe4\x46\xe1\x52\xf6\x9e\xe7\x54\x64\xd0\x13\xe5\xb9\x94\x54\x9f\x86\x6e\xaf\xa9\xb0\x50\x3f\xd9\x7b\xb9\x3e\xc2\x0d\xaa\x69\x4a\x65\xbc\x3f\x33\x20\x61\x66\x4d\x28\x14\xa3\x22\x7c\xe2\x32\xdc\x79\x3d\x36\x3c\x6f\x80\x0e\x48\x90\xcf\xff\xda\xfc\x33\x55\x31\xcc\x07\xd4\x0b\x48\x50\xcf\xdb\xf3\x4c\xf2\xe7\x28\x0c\x43\x66\x79\xd8\x0e\xa5\xa9\x55\x6c\xdb\xdc\x63\x20\xe1\x16\x4d\xb1\xa8\xf7\x22\xd3\x51\x18\x5e\x4c\xfd\xbf\x68\x5f\xf8\xab\xf5\xce\x67\x3c\x9e\x7f\xf7\xfd\x3f\x7e\xfd\x79\xf1\x78\x73\x7b\xb5\x78\xbc\xb9\xbc\x5e\xfc\xd0\x5c\xc3\xe8\x2f\x5c\xc0\x3c\x64\x60\xd0\x86\x8c\x4e\x99\xc1\xd6\x73\x30\xd8\x5c\x81\xc2\x76\x97\x6c\xa1\xe8\xae\xd8\x42\xd1\x5c\x51\x1a\xca\x55\x97\xf9\x77\xdf\xaf\xae\x17\xd7\x8f\x9f\xef\x97\x8f\x57\x77\xcb\xdf\x16\x77\x7d\x48\x55\x45\xff\xea\x47\xab\x79\xb4\x2f\xd5\xee\x03\x32\x6f\xba\xa6\xac\x66\x6d\xfd\x26\x9d\xf8\xf8\xc5\xa8\xb4\x1d\x22\x6b\x0e\x22\xae\x68\x49\xf3\xd3\xe9\x5f\xed\x87\x7e\xd3\x8a\xe2\x26\xf2\xe9\x37\x75\xd6\x75\x05\xa9\x87\xe6\x61\x71\x77\xbd\xbc\xb9\x7c\x58\xde\xde\x3c\x7e\xbd\xfd\xf2\xb8\xba\x7c\xf8\x7b\x17\x53\x44\x42\x4c\x75\x88\x60\xd2\xea\x4b\x91\x40\xa8\xa4\x77\xd4\x90\xad\xfa\x47\xf5\x73\x6a\xb7\x86\xa7\x34\x81\x88\xf8\x07\xdd\xc0\x8d\x18\x95\xd4\x14\xed\xb5\xab\x4c\x88\x95\x12\x9c\x15\x11\x59\xae\x6f\x14\xae\x0c\x58\x90\x7b\x87\x37\xfa\x47\x27\xfe\x77\x4d\xb6\xd6\x61\x97\x5e\xab\x4e\xcf\x6d\x9e\xb4\xef\xdf\xe5\xc7\x02\xcb\x0c\xc7\xe2\xb3\x1b\xcf\x9e\xb1\x5d\xbf\x68\x7c\x2b\x45\x71\xa7\x94\x8f\x49\x5b\x58\x84\x34\x22\x68\xb2\xbd\x17\x1a\x26\xbd\x06\x6b\x9d\x4a\xde\x67\xa3\xf6\x2e\x47\xa1\x6b\xc7\x10\x5b\x1a\xa4\xee\x4e\xb5\xdf\x47\x77\x0f\x7e\x9d\x93\x3e\x51\x0e\xed\xb4\xbc\xb7\x4f\x8b\x2c\xe1\x32\x70\x29\x0e\x18\xc4\xdc\x1c\xd8\x8b\xa9\xee\xed\xc5\x54\x37\x76\x04\x84\x9a\xa4\x85\xba\x5b\x86\xbc\xcf\x69\x1c\xbb\xd2\x3c\x3f\xa6\xd0\x04\x6b\xa0\x98\x19\x08\x12\x8a\x60\xe7\x0f\x4a\x2b\xa1\x92\x62\xde\x32\xb4\x5b\x67\x9d\xef\x30\xc0\x83\x0b\x90\xa7\xa0\x32\x9c\xff\x2d\x6d\xdd\xae\x9c\xbe\x03\xe5\x93\x7c\xd6\x8d\xda\xdf\x33\x5a\xb8\x61\x66\xfb\x93\xad\x41\x37\x28\x7f\x94\xff\xe8\xca\xde\x59\xe1\x3b\x34\x42\x9c\x14\xc7\xb3\x37\x8f\xe3\xa3\x42\xf2\x84\xc0\x3a\x88\xc7\x64\xf2\xd2\xde\x28\xe9\xd0\x74\x30\xf8\x47\xbf\x5a\x30\xe5\xfc\xb0\x3b\xa9\x39\x51\x8d\x71\xb2\xbd\x1e\x8d\x06\x0b\xa9\xc6\xe2\x8a\x9b\x88\xc8\x4c\x88\x6e\x5d\x39\x80\x3e\x18\x4d\x37\x0b\xcc\x40\xcb\xc4\xe5\x9d\x0e\xb6\xdd\xe6\xf2\xa9\x1d\x80\xf4\xaf\x7f\x77\x00\xd5\xb9\x36\x4e\xd4\xaf\x28\xa4\x4a\xbe\x21\x4d\x77\xcd\x66\x9c\xa0\x9f\x48\xc9\xab\x03\xcf\x81\xf6\xa1\x4c\xbb\x81\xfb\x1d\x39\xf6\xab\x13\xc5\x18\x5c\x0a\x5d\x53\x49\x13\x30\x73\x91\xa7\x2f\xb1\xc8\x96\xde\x23\xfc\xb1\x57\xcb\x8f\x22\x81\xbb\xa4\xed\x50\xcb\x6a\xd7\xea\xf6\xea\x71\xb9\xfa\xc1\x33\xcc\xd9\x11\x8c\x74\xa0\x1a\xf4\xd9\xe9\xbb\x30\x4f\xeb\x26\x68\x57\x2d\xe7\x61\x4e\x4d\x28\xf8\x53\x78\x04\x03\x3d\x8d\xaf\x3a\x6d\x57\x60\x18\x48\xa4\x09\xcc\x67\x6d\x35\xff\xd7\xd9\x6c\x23\x00\xde\x01\x0a\x52\xcc\xec\x54\xab\xb8\x21\xed\xd5\xc8\xf0\x2b\x12\xf4\xff\x2b\x5e\xad\x0d\xcf\xb9\x80\x04\xe2\x4e\xff\x6f\x31\x80\x8b\xb7\xa7\xd8\x75\x8a\x6e\xb3\x27\x10\x80\x61\xc9\x06\xac\xff\xdb\x48\x40\xf0\xdf\x6d\xb7\x59\x4f\xb9\xdf\x28\x4d\x13\x2f\x34\x22\x3f\xf3\x98\x1b\xf0\x35\x9f\x8a\xbe\x5d\xdc\x7a\x5f\xac\x46\x58\x77\x0f\x48\xf9\x4a\xf6\x5c\x99\x6e\xfb\x88\xb4\xe1\xc9\xc2\x65\xe4\xd8\x54\x11\x43\xde\xdb\x13\x43\x3e\x22\xc7\x16\x7d\x29\xb6\x18\x45\x36\x40\x30\x5f\x1c\x59\x6a\xe3\x8d\x64\xe2\x19\x26\x74\x87\xf9\x6a\x7e\xda\xe8\x53\x79\xb0\xee\x54\x65\x5c\xea\x56\x43\xe8\x46\xdb\x81\x92\x3f\xd8\x5d\x07\x47\xab\xf6\xaa\xc1\x1a\xff\x5a\x83\xfd\xc0\x88\xe4\xe3\xa6\x2c\x21\x3b\xad\x4d\x94\xcf\xa6\x3f\x9e\x39\x2c\x75\xcf\xfa\xf0\x11\xe7\xc5\x08\x6c\x3a\xfb\xd0\xc4\x5e\x46\xc2\xfe\x8c\xe6\x17\x9c\x7b\x30\xd5\x3b\xd4\xd2\x15\x87\x26\x9a\x8d\xb2\xa5\xe4\x66\x4d\x1d\xaf\x68\xa3\xb9\x51\xbe\x23\xb8\xf2\xd9\xa0\x4c\x71\x6b\x3e\xd7\xaf\xdc\x47\xad\x70\x0e\x90\xc7\x9a\xc1\x85\x27\xcb\x3f\x60\xc7\xb3\xcc\x31\x56\xe0\x8f\x43\x33\x58\xd6\x4f\xc3\xd2\xae\xf1\xc7\x89\xed\x54\xf6\xe0\x60\xed\x3e\x72\x44\xf5\x1b\xbb\xe3\xe9\x11\x4a\xbc\x42\x3c\x1d\xa8\xb0\x23\xc2\xdb\x1d\xe8\x38\x29\xed\x0e\x35\x7a\x7c\xbb\x5d\x1d\x99\x14\x55\x3b\xeb\x8e\xeb\xed\xdf\x42\x84\xf9\xec\x09\x90\xee\xde\x91\xdf\x2f\xaf\x4a\xca\xf5\x36\xd3\x7b\xc3\x29\xf5\x50\x59\xfe\xea\xe2\xae\x7a\x21\x13\x11\xff\x5a\xc2\xbf\xa8\x8c\x97\x72\xad\x6e\xa5\xaf\x8a\xbb\x7a\x59\x96\x9c\xaf\x7c\x0d\xac\x60\x02\xae\x55\x5c\xff\x88\x63\xb5\xfb\x0d\x92\xff\x73\xa1\x37\x90\x82\xa1\x62\xf2\x9f\x00\x00\x00\xff\xff\xb2\xb2\xba\xec\x86\x29\x00\x00") func deployKubernetes117LvmPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -119,12 +119,12 @@ func deployKubernetes117LvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.17/lvm/pmem-csi.yaml", size: 10223, mode: os.FileMode(420), modTime: time.Unix(1599471424, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.17/lvm/pmem-csi.yaml", size: 10630, mode: os.FileMode(420), modTime: time.Unix(1599822736, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes118DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5f\x6f\xe3\xb8\x11\x7f\xf7\xa7\x60\xb7\xf7\x70\x07\x54\x56\x72\x45\x81\x83\x00\x3f\x64\x37\xbe\xad\xd1\x8d\x63\x24\xb9\x7b\x0d\x68\x6a\x2c\xb3\xa6\x48\x1e\x39\xf2\x46\x57\xf4\xbb\x17\xa4\x24\x5b\xff\x2c\xff\xb9\x64\xd3\x73\x1e\x22\x8b\x1c\xce\x8f\x33\xc3\x99\xdf\x48\xfe\x2b\xf9\x0c\x12\x0c\x45\x88\xc9\x57\x8e\x6b\xf2\x21\xa5\x1b\x20\x9b\xcc\xa2\x4a\xf9\xef\xf0\xe1\x6f\x24\x56\x44\x2a\x24\x10\x73\xfc\xcb\x68\x44\x35\xff\x15\x8c\xe5\x4a\x46\x64\x7b\x3d\xda\x70\x19\x47\xe4\x11\xcc\x96\x33\xb8\x61\x4c\x65\x12\x47\x29\x20\x8d\x29\xd2\x68\x44\x88\xa0\x4b\x10\xd6\x5d\x11\xa2\x53\x48\x03\x66\xf9\x98\x4b\x04\x31\x66\x2a\x0d\x63\xd0\x42\xe5\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x22\x44\xd2\x14\xa2\x9d\x60\xc0\x94\x44\xa3\x84\x00\x53\x8e\x59\x4d\x19\x44\x24\x86\x15\xcd\x04\x8e\x82\x20\x68\x20\x34\x4b\xca\xc6\x34\xc3\xb5\x32\xfc\x77\xea\xd6\x1c\x6f\x7e\xb2\x63\xae\xc2\x1d\xf6\x07\x25\xe0\x0d\x11\xc3\x0b\x82\x91\x54\xb8\x49\x5b\xee\x50\x81\x09\xd8\x2a\xe9\xc7\x6f\x32\x01\x36\x1a\x05\x84\x6a\xfe\xd9\xa8\x4c\x7b\x1c\x01\xf9\xf0\x61\x44\x88\x01\xab\x32\xc3\xa0\xbc\x07\x32\xd6\x8a\x4b\xb4\x23\x42\xb6\x60\x96\xe5\xed\x04\xd0\xff\xff\x4a\x91\xad\xfd\x95\xe0\xb6\xb8\x15\x83\x00\x04\x7f\x99\xe9\x98\x96\x97\xcc\x80\xbb\xec\xe8\x64\x4a\x99\x98\xcb\xba\xd5\xba\x20\x04\x50\x0b\x6f\x85\xc0\xa2\x32\x34\x81\x83\xca\x99\xe5\xe5\x14\x46\x35\x65\x1c\xf9\x21\x28\x3b\x00\x7b\x4c\xa5\xd2\x16\x14\xbd\x1b\x2f\xa1\x9e\xe4\x0a\xad\xe2\x1e\xc5\x1d\x51\xaa\xb5\xed\x0a\x1b\xd0\x82\x33\x6a\xa1\xcf\x93\x17\x04\xf4\x27\x91\x59\x04\xf3\x1e\x71\x6d\x32\x29\xc1\x9c\x15\xc5\xda\xed\xcc\x22\x48\xdc\x2a\x91\xa5\x97\x39\xf0\x2c\x5f\xb5\x14\x32\x41\x79\x7a\xba\xd6\x32\x56\xce\x8e\xd5\x2a\x50\x05\xb5\x07\x0f\x4c\x4b\xdd\x69\x59\x60\x0b\x9d\x14\x70\x66\xb4\x77\xf7\x22\xa9\xb6\x6b\x85\xe3\x63\x9b\x2a\x5d\x56\x4e\x1f\xda\xd5\x6b\xe9\x70\x05\xa0\xbb\xe1\x23\xaa\x8e\xa7\x11\xa9\xe2\xd7\x74\xca\x1f\x58\xee\x34\x73\x50\x44\xca\xd6\xe9\x11\x4b\xd4\xd4\x5c\x56\x17\x3f\x72\x19\x73\x99\xbc\x65\x41\xb7\xbc\x99\x41\x94\x80\x81\xf2\xa8\x04\x3c\xc0\xca\x29\xaf\xcc\x36\xb0\x97\x11\x21\xb5\x12\x7f\x7a\x65\xb6\xd9\xf2\xdf\xc0\xd0\x67\xb0\x5e\x7e\xf3\x2d\x68\x49\x2d\x8b\xbf\x8f\x17\x2e\x35\x76\xbd\xfc\x9c\x55\x35\xde\xde\xec\x6d\xbe\xfa\x8e\x44\xd5\x6a\x60\x4e\x97\x56\x06\x77\x24\xc2\x60\x44\xae\xaf\xae\xae\xae\x3c\x08\xa4\x26\x01\x5c\x34\xee\x5a\x10\xc0\x50\x99\x02\x26\xd5\xfa\x90\xca\x0b\xf6\xf0\x8e\xd6\x4a\x01\x0d\x67\xf6\x7c\x53\x5d\xf7\x9a\xea\xfa\xad\x4d\x45\x08\xe6\x1a\x22\x32\x57\x31\x38\xad\x1d\xdb\x39\xa6\xb7\x3f\xcb\x8f\x48\x11\x56\x99\x78\x84\xf7\xec\x8d\x2a\x3b\x56\x7c\x33\x22\xd7\x1d\x33\xa5\xae\x5a\x7c\xa9\xc1\x3a\x62\xb9\x8b\x70\xdb\x22\x9e\xe6\x03\xf0\x11\x52\x2d\x28\x42\x89\xaa\x66\x33\xf7\x11\x0d\x80\x47\x21\x5e\x04\xf2\xa0\xe0\x57\x58\xae\x95\xda\x44\x84\x27\x52\x19\xf0\x53\x2b\xd3\x7a\x30\xab\x15\x97\x1c\xf3\x3d\x3c\xc7\x06\x6e\x3a\x77\x9d\x23\x7e\xcb\xb8\x81\xf8\x36\x33\x5c\x26\x8f\x6c\x0d\x71\x26\xb8\x4c\x66\x7e\xe1\xf2\xf6\xf4\x05\x58\xe6\x30\xd5\x25\x8b\x35\x1f\x4b\xd7\x3d\x81\x49\x6d\x73\x38\x28\x3c\x39\x7d\xd1\x06\xac\x0b\xc9\xd6\xb8\x9b\xb1\x81\x3c\xea\xdb\x60\x8f\xfd\xaa\x8f\xd2\x60\xa8\x0b\x16\x32\x57\x38\x93\x9d\xf1\x2d\x15\x19\x74\x54\x79\x96\x24\xd5\x87\xbe\xdb\x2b\x2a\x2c\x54\x23\x4e\x33\xe5\x12\xcc\x6e\x09\xd7\x85\xa6\x29\x95\xf1\x7e\xcd\x80\x84\x99\x35\xa1\x50\x8c\x8a\x70\xc9\x65\xb8\x73\x7c\x6c\xf8\xb6\x06\x3a\x20\xc1\x76\xf2\xf7\xfa\xd7\x54\xc5\x30\xe9\xd9\x5e\x40\x82\xaa\x99\x9e\x64\x92\xbf\x44\x61\x18\x32\xcb\xc3\x66\x34\x8d\xad\x62\x9b\xba\x8c\x81\x84\x5b\x34\xf9\xb4\x92\x45\xa6\xa3\x30\xbc\x1a\xfb\xbf\x68\x9f\xc9\x2b\xf5\x45\x96\xfb\xe2\xbb\x8f\x49\xb4\x4f\x5f\xe5\xb8\xf3\x29\x8f\x27\xdf\x7d\xff\xaf\x5f\x3e\x4e\x9f\xe7\xf7\xb7\xd3\xe7\xf9\xcd\xdd\xf4\x87\xfa\x1c\x46\x7f\xe6\x02\x26\x21\x03\x83\x36\x64\x74\xcc\x0c\x36\xc6\xc1\x60\x7d\x06\x0a\xdb\x9e\xb2\x81\xbc\x3d\x63\x03\x79\x7d\x46\x61\x48\x97\x42\x26\xdf\x7d\xbf\xb8\x9b\xde\x3d\x7f\x7a\x9c\x3d\xdf\x3e\xcc\x7e\x9d\x3e\xb4\x20\x81\xdc\xd6\x5d\x53\xa4\xa4\x26\xfe\x51\x2b\x3e\x7e\x36\x2a\x6d\x86\xc8\x8a\x83\x88\x4b\xaa\x51\xff\xb4\xaa\x51\x73\xd0\x0b\x2d\x28\xae\x23\x7f\xfc\xc6\xce\x7a\x2e\xa5\x74\xd0\x3c\x4d\x1f\xee\x66\xf3\x9b\xa7\xd9\xfd\xfc\xf9\xcb\xfd\xe7\xe7\xc5\xcd\xd3\x3f\xdb\x98\x22\x12\x62\xaa\x43\x04\x93\x96\x4f\x3c\x02\xa1\x92\xce\x52\x7d\xb6\xe8\x2e\xd5\x3d\x53\xbb\x39\x3c\xa5\x09\x44\xc4\x0f\xb4\x03\x37\x62\x54\x52\x93\x37\xe7\x2e\x32\x21\x16\x4a\x70\x96\x47\x64\xb6\x9a\x2b\x5c\x18\xb0\x20\xf7\x0e\xad\x15\x81\x56\xfc\x5b\x60\x99\xe1\x98\x7f\x72\x2d\xd3\x0b\x36\x33\x0f\x8d\xef\xa5\xc8\x1f\x94\xf2\xd1\x62\x73\x8b\x90\x46\x04\x4d\xb6\xb7\x5f\xcd\x18\x77\x60\xad\x03\xe3\xad\x3d\x68\xa9\xa2\x3d\xb9\x73\x7c\xcd\xd6\x03\x23\x75\x77\x4a\x79\x1f\x77\x35\x34\xc5\x16\xaa\xd3\xe4\x43\xf8\x90\xa4\xe5\x1d\x39\x2d\xb2\x84\xcb\xc0\x1d\x4e\xc0\x20\xe6\xe6\x80\x2c\xa6\xba\x23\x8b\xa9\xae\x49\x04\x84\x9a\xa4\x81\xba\x9d\x40\xbc\xb7\x68\x1c\xbb\xa4\x3a\x39\x25\x45\x04\x2b\xa0\x98\x19\x08\x12\x8a\x60\x27\x4f\x4a\x2b\xa1\x92\x7c\xd2\x30\xb4\x9b\x67\x5d\x6a\xc0\x00\x0f\x4e\x40\x9e\x82\xca\x70\xf2\x8f\x4e\x30\xfd\x96\xd1\xdc\xb5\x0e\x9b\x9f\x6c\x85\xa8\xc6\xae\xa3\xed\x8f\x2e\x1b\x5d\x14\x55\x7d\x6c\xfd\xd5\xc2\xeb\xa4\x48\x39\xc3\xdf\x07\xf1\x98\x4c\xde\xd8\xb9\x92\x0e\x4d\x0b\x83\x1f\xfa\xc5\x82\x29\x18\xf6\x6e\xa5\x7a\xdb\x31\x44\x53\xf6\xfb\xa8\x55\x2c\x48\x35\xe6\xb7\xdc\x44\x44\x66\x42\xb4\x0f\xea\x01\xf4\xc1\xe0\x29\xb0\xc0\x0c\x34\x4c\x5c\xdc\x69\x61\xdb\x09\x17\xa3\xb6\x07\xd2\x7f\xfe\x3b\xea\x3f\x02\xc3\x0c\xf6\x96\x42\xaa\xe4\xdb\xf2\x57\x97\xc0\x87\x99\xeb\x99\x5c\xb5\x5c\xf0\x42\x74\x7f\x84\x82\xd6\x34\x7f\x5b\xf2\xf9\xea\x0c\x2a\x06\x77\x14\xee\xa8\xa4\x09\x98\x49\x81\xf5\x18\xc3\x6a\xec\x7e\x80\x5b\x75\xb2\xe5\x49\x04\x68\x77\xfe\x5a\xb4\xab\x94\x5a\xdc\xdf\x3e\xcf\x16\x3f\x78\xf6\x75\x7d\x02\x5b\xeb\x39\xd8\x5d\xe6\xf6\x4d\x58\x97\x75\x5d\xa2\x4b\x7c\x93\x70\x4b\x4d\x28\xf8\x32\x3c\xc6\xbe\xce\xe3\x6a\x01\x09\xdc\x6e\x17\x60\x18\x48\xa4\x09\x4c\xae\x6b\xdb\xfc\x13\x30\xb9\x9a\x83\xbf\x01\x14\xa4\x98\xd9\xb1\x56\x71\x4d\xdb\xab\x11\xc1\x57\x24\xa7\xff\x27\x9c\x52\x1b\xbe\xe5\x02\x12\x88\x5b\x45\xb6\x51\x66\xaf\xde\x9e\x5e\x56\x87\x67\x93\x2d\x41\x00\x86\x45\xc9\xb5\xfe\xbb\x91\x80\xe0\x1f\xb4\x36\xa9\x45\x21\x6f\x94\xa6\x89\x57\x1a\x91\x8f\xbc\x48\x76\x5c\x49\x2a\x3a\x24\xc4\xcf\xf7\x69\x64\x80\x71\x76\x80\x14\x6f\x09\x2f\xd5\xe9\xc4\x07\xb4\xf5\xb3\x6a\x77\x96\x86\x18\x75\x0c\xdb\x8e\x4c\x0c\xdb\x01\x3d\x36\xef\x6a\xb1\xf9\x20\xb2\x1e\x16\x77\x94\xae\x57\xc6\x1b\x38\x43\x17\x98\xd0\x2d\xe6\xf3\xec\x79\xb4\xbf\xf4\x60\x55\x43\x8a\xb8\xd4\x8d\x54\xdd\x8e\xb6\x03\xc9\xb8\xb7\xee\xf5\xb6\x15\xcd\x59\xbd\xd9\xf9\xb5\xda\xd1\x9e\x0e\xc2\xc7\x4d\x71\xf8\x77\xbb\x36\xd1\xf6\x7a\xfc\xe3\x85\xbd\x44\x7b\xad\x77\xef\x23\x8e\x46\x60\xdd\xd9\x87\xba\xd5\x22\x12\xf6\x6b\xd4\x1f\xcb\xed\xc1\x94\xef\xf4\x0a\x57\x1c\x6a\x1b\xd6\xca\x16\x9a\xeb\x39\x75\x38\xa3\x0d\x9e\x8d\xe2\x09\xf5\xad\x3f\x0d\xca\xe4\xf7\xe6\x53\xf5\x0a\x78\xd0\x0a\x97\x00\x79\xae\xb8\x55\x78\xb6\xfe\x03\x76\xbc\xc8\x1c\x43\x09\xfe\x34\x34\xbd\x69\xfd\x3c\x2c\xcd\x1c\x7f\x9a\xda\x56\x66\x0f\x0e\xe6\xee\x13\xfb\x40\x2f\xd8\xee\x01\x4f\xd8\xc4\x2b\xc4\xd3\x81\x0c\x3b\xa0\xbc\x59\x81\x4e\xd3\xd2\xac\x50\x83\xcb\x37\xcb\xd5\x89\x87\xa2\x2c\x67\xed\x9e\xb8\xf9\x6e\x3e\xdc\x5e\x2f\x01\xe9\xee\x85\xed\xe3\xec\xb6\x20\x4b\x6f\xd6\x22\xd7\xfc\x52\x35\x7d\xc5\x0f\x01\x1e\xca\x37\x09\x11\xf1\xcf\xd3\xfd\xcb\xb2\x78\x26\x57\xea\x5e\xfa\xc4\xb8\x4b\x99\x45\xd6\xf9\xc2\x57\xc0\x72\x26\xe0\x4e\xc5\xd5\xef\x0a\x16\xbb\x9f\xc5\xf8\xaf\x53\xbd\x86\x14\x0c\x15\xa3\xff\x05\x00\x00\xff\xff\x56\x33\x4f\x09\x22\x28\x00\x00") +var _deployKubernetes118DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\xa6\xf7\x70\x07\x54\x56\x7c\x45\x81\x83\x00\x3f\xe4\x36\xbe\xad\xd1\x4d\x62\x24\xb9\x7b\x0d\x18\x6a\x2c\xb3\xa6\x48\x1e\x39\xf2\x46\x57\xf4\x7f\x2f\x48\x7d\x58\x5f\x56\x6c\x37\xd9\xa0\xf5\x3e\xac\x22\x71\x38\xbf\xf9\xe0\xcc\x6f\x2c\xff\x99\x7c\x06\x09\x86\x22\xc4\xe4\x2b\xc7\x0d\xb9\x48\xe9\x16\xc8\x36\xb3\xa8\x52\xfe\x07\x5c\xfc\x85\xc4\x8a\x48\x85\x04\x62\x8e\x7f\x9a\x4c\xa8\xe6\xbf\x81\xb1\x5c\xc9\x88\xec\x66\x93\x2d\x97\x71\x44\x1e\xc0\xec\x38\x83\x2b\xc6\x54\x26\x71\x92\x02\xd2\x98\x22\x8d\x26\x84\x08\xfa\x0c\xc2\xba\x2b\x42\x74\x0a\x69\xc0\x2c\x9f\x72\x89\x20\xa6\x4c\xa5\x61\x0c\x5a\xa8\x3c\x05\x89\x11\x89\xb9\x01\x86\x81\x36\x2a\xce\x18\x72\x25\x27\x84\x48\x9a\x42\x54\x0b\x06\x4c\x49\x34\x4a\x08\x30\xe5\x33\xab\x29\x83\x88\xc4\xb0\xa6\x99\xc0\x49\x10\x04\x2d\x84\xe6\x99\xb2\x29\xcd\x70\xa3\x0c\xff\x83\xba\x3d\xa7\xdb\x9f\xec\x94\xab\xb0\xc6\x7e\xaf\x04\xbc\x23\x62\x78\x41\x30\x92\x0a\xb7\x68\xc7\x1d\x2a\x30\x01\x5b\x27\xc3\xf8\x4d\x26\xc0\x46\x93\x80\x50\xcd\x3f\x1b\x95\x69\x8f\x23\x20\x17\x17\x13\x42\x0c\x58\x95\x19\x06\xe5\x3d\x90\xb1\x56\x5c\xa2\x9d\x10\xb2\x03\xf3\x5c\xde\x4e\x00\xfd\xff\x5f\x29\xb2\x8d\xbf\x12\xdc\x16\xb7\x62\x10\x80\xe0\x2f\x33\x1d\xd3\xf2\x92\x19\x70\x97\x3d\x9d\x4c\x29\x13\x73\xd9\xf4\x5a\x1f\x84\x00\x6a\xe1\xbd\x10\x58\x54\x86\x26\x70\x50\x39\xb3\xbc\x5c\xc2\xa8\xa6\x8c\x23\x3f\x04\xa5\x06\xb0\xc7\x54\x2a\xed\x40\xd1\xf5\xf3\x12\xea\x51\xa1\xd0\x2a\x1e\x50\xdc\x13\xa5\x5a\xdb\xbe\xb0\x01\x2d\x38\xa3\x16\x86\x22\x79\x46\x42\x7f\x12\x99\x45\x30\x1f\x91\xd7\x26\x93\x12\xcc\x49\x59\xac\x9d\x65\x16\x41\xe2\x4e\x89\x2c\x3d\x2f\x80\x27\xc5\xaa\xa3\x90\x09\xca\xd3\xe3\xb5\x96\xb9\x72\x72\xae\x56\x89\x2a\xa8\x3d\x78\x60\x3a\xea\x8e\xab\x02\x3b\xe8\x95\x80\x13\xb3\xbd\x6f\x8b\xa4\xda\x6e\x14\x4e\x5f\x33\xaa\x0c\x59\xb9\x7c\xcc\xaa\xb7\xd2\xe1\x1a\x40\xdf\xe0\x57\x54\xbd\x5e\x46\xa4\x8a\xdf\x32\x28\xff\xc5\x76\xc7\xb9\x83\x22\x52\xb6\x49\x5f\xf1\x44\x43\xcd\x79\x7d\xf1\x67\x2e\x63\x2e\x93\xf7\x6c\xe8\x96\xb7\x2b\x88\x12\x30\xd2\x1e\x95\x80\x7b\x58\x3b\xe5\x95\xdb\x46\x6c\x99\x10\xd2\x68\xf1\xc7\x77\x66\x9b\x3d\xff\x13\x18\xfa\x0a\x36\xc8\x6f\xbe\x05\x2d\x69\x54\xf1\x8f\x89\xc2\xb9\xce\x6e\xb6\x9f\x93\xba\xc6\xfb\xbb\xbd\xcb\x57\x3f\x90\xa8\x5a\x0d\xcc\xe9\xd2\xca\x60\x4d\x22\x0c\x46\x64\x76\x79\x79\x79\xe9\x41\x20\x35\x09\xe0\xaa\x75\xd7\x82\x00\x86\xca\x14\x30\xa9\xd6\x87\x54\x9e\x61\xc3\x07\x7a\x2b\x05\x34\x9c\xd9\xd3\x5d\x35\x1b\x74\xd5\xec\xbd\x5d\x45\x08\xe6\x1a\x22\x72\xab\x62\x70\x5a\x7b\xbe\x73\x4c\x6f\x7f\x96\x1f\x90\x22\xac\x33\xf1\x00\x1f\x39\x1b\x55\x7e\xac\xf8\x66\x44\x66\x3d\x37\xa5\xae\x5b\x7c\x69\xc0\x7a\xc5\x73\x67\xe1\xb6\x45\x3e\xdd\x8e\xc0\x47\x48\xb5\xa0\x08\x25\xaa\x86\xcf\x3c\x24\x29\x15\xfa\xe2\x53\xa3\x1c\x04\x62\x99\xa1\x2e\x4c\x6e\x6b\xca\x25\x18\x5b\xae\x16\x2d\x0b\x5f\xb5\xf1\x2c\x2b\x0f\x0a\x7e\x85\xe7\x8d\x52\xdb\x88\xf0\x44\x2a\x03\x7e\x69\x15\x1b\x0f\x66\xbd\xe6\x92\x63\xbe\x87\xe7\xe8\xc4\x55\xef\xae\x8b\xe4\xef\x19\x37\x10\x5f\x67\x86\xcb\xe4\x81\x6d\x20\xce\x04\x97\xc9\xd2\x6f\x5c\xde\x5e\xbc\x00\xcb\x1c\xa6\xa6\x64\xb1\xe7\x43\x19\xfb\x47\x30\xa9\x6d\x3f\x0e\x8a\x54\x58\xbc\x68\x03\xd6\xb6\x3d\x5d\xad\xd8\x42\x1e\x0d\x19\x38\xe0\xbf\xea\xa3\x34\x18\xea\xb2\x8d\xdc\x2a\x5c\xca\xde\xf3\x1d\x15\x19\xf4\x54\x79\x9a\x25\xd5\xc5\xd0\xed\x35\x15\x16\xaa\x27\xfb\x40\x57\x5b\xb8\x31\x36\x4d\xa9\x8c\xf7\x7b\x06\x24\xcc\xac\x09\x85\x62\x54\x84\xcf\x5c\x86\x75\xe0\x63\xc3\x77\x0d\xd0\x01\x09\x76\xf3\xbf\x36\xff\x4c\x55\x0c\xf3\x01\xf3\x02\x12\x54\xd3\xf8\x3c\x93\xfc\x25\x0a\xc3\x90\x59\x1e\xb6\xb3\x69\x6a\x15\xdb\x36\x65\x0c\x24\xdc\xa2\xc9\x17\x95\x2c\x32\x1d\x85\xe1\xe5\xd4\xff\x8b\xf6\xad\xa0\x5c\xef\x62\xc6\xe3\xf9\x77\xdf\xff\xe3\xd7\x9f\x17\x4f\xb7\x77\xd7\x8b\xa7\xdb\xab\x9b\xc5\x0f\xcd\x35\x8c\xfe\xc2\x05\xcc\x43\x06\x06\x6d\xc8\xe8\x94\x19\x6c\x3d\x07\x83\xcd\x15\x28\x6c\x77\xc9\x16\xf2\xee\x8a\x2d\xe4\xcd\x15\x85\xa3\x5c\x8d\x99\x7f\xf7\xfd\xea\x66\x71\xf3\xf4\xe9\x61\xf9\x74\x7d\xbf\xfc\x6d\x71\xdf\x87\x54\x56\xf7\x2f\x7e\xea\x9a\x47\xfb\xb2\xed\x3e\x20\x77\xcd\xd0\x14\x35\xad\x6d\xdf\xa4\x93\x1f\xbf\x18\x95\xb6\x53\x64\xcd\x41\xc4\x25\x57\x69\x7e\x3a\xed\xac\xfd\xd0\x0b\xad\x28\x6e\x22\x7f\xfc\xa6\xce\xbb\xae\x26\xf5\xd0\x3c\x2e\xee\x6f\x96\xb7\x57\x8f\xcb\xbb\xdb\xa7\x2f\x77\x9f\x9f\x56\x57\x8f\x7f\xef\x62\x8a\x48\x88\xa9\x0e\x11\x4c\x5a\x7e\x65\x12\x08\x95\xf4\xb6\x1a\xf2\x55\x7f\xab\xfe\x99\xaa\xd7\xf0\x94\x26\x10\x11\xff\xa0\x9b\xb8\x11\xa3\x92\x9a\xbc\xbd\x76\x95\x09\xb1\x52\x82\xb3\x3c\x22\xcb\xf5\xad\xc2\x95\x01\x0b\x72\x1f\xf0\x46\x17\xe9\xe4\x7f\xdd\x70\x2b\x1b\xea\xe3\xb5\xea\xf4\xdf\xe6\x4e\xfb\x5e\x5e\x7c\x2c\xb0\xcc\x70\xcc\x3f\xb9\xc9\xed\x05\xdb\xf5\x8b\xc6\x77\x52\xe4\xf7\x4a\xf9\x9c\xb4\xb9\x45\x48\x23\x82\x26\xdb\x47\xa1\xe1\xd2\x1b\xb0\xd6\x99\xe4\x63\x36\xea\xef\x62\x4a\xba\x71\xb4\xb1\x65\x41\xea\xee\x94\xf2\x3e\xbb\x7b\xf0\xab\x33\xe9\x0f\xca\x21\x49\xcb\x7b\x72\x5a\x64\x09\x97\x81\x3b\xe2\x80\x41\xcc\xcd\x01\x59\x4c\x75\x4f\x16\x53\xdd\x90\x08\x08\x35\x49\x0b\x75\xb7\x0c\xf9\x98\xd3\x38\x76\xa5\x79\x7e\x4c\xa1\x09\xd6\x40\x31\x33\x10\x24\x14\xc1\xce\x1f\x95\x56\x42\x25\xf9\xbc\xe5\x68\xb7\xce\xba\xd8\x61\x80\x07\x17\x20\x4f\x41\x65\x38\xff\x5b\xda\xba\x5d\x06\xbd\x06\xe5\x0f\xf9\xac\x9b\xb5\xbf\x67\x34\x77\x43\xce\xf6\x27\x5b\x81\x6e\xcc\x01\xd1\xee\x47\x57\xf6\xce\x4a\xdf\xa1\xb9\xe2\xa4\x3c\x9e\xbd\x7b\x1e\x1f\x95\x92\x27\x24\xd6\x41\x3c\x26\x93\x57\xf6\x56\x49\x87\xa6\x83\xc1\x3f\xfa\xd5\x82\x29\x26\x8a\x7a\xa7\xe6\x98\x35\x46\xcb\xf6\x76\x34\x1a\x2c\xa4\x1a\xf3\x6b\x6e\x22\x22\x33\x21\xba\x75\xe5\x00\xfa\x60\xf4\xb8\x59\x60\x06\x5a\x2e\x2e\xee\x74\xb0\xd5\xc2\xc5\x53\x3b\x00\xe9\x5f\xff\xee\x00\xaa\xce\xda\x38\x63\xbf\xa6\x90\x2a\xf9\xbe\x7c\xdd\xf5\x9b\x71\xa6\x7e\x22\x37\x2f\x37\x3c\x13\xdd\x87\x52\xee\x06\xf4\x6f\x4b\xb6\xdf\x9c\x31\xc6\xe0\xce\xd2\x0d\x95\x34\x01\x33\x2f\xb0\xbe\xc6\x28\x5b\xd6\x8f\x70\xc9\x5e\x5d\x3f\x8a\x10\xd6\x07\xb8\x43\x33\x4b\xa9\xd5\xdd\xf5\xd3\x72\xf5\x83\x67\x9b\xb3\x23\xd8\xe9\x40\x65\xe8\x33\xd5\x6f\xc2\x42\xad\x1b\xab\x5d\xe5\x9c\x87\x3b\x6a\x42\xc1\x9f\xc3\x23\xd8\xe8\x69\xdc\xd5\x59\xbb\x02\xc3\x40\x22\x4d\x60\x3e\x6b\x9b\xf9\xbf\xce\x6c\x1b\x09\xf0\x0d\xa0\x20\xc5\xcc\x4e\xb5\x8a\x1b\xda\xde\x8c\x18\xbf\x21\x59\xff\xbf\xe2\xd8\xda\xf0\x1d\x17\x90\x40\xdc\xe1\x02\x2d\x36\x70\xf9\xfe\x74\xbb\x3a\xa2\xdb\xec\x19\x04\x60\x58\x30\x03\xeb\xff\x36\x12\x10\xfc\xf7\xdf\x6d\x06\x54\xc8\x1b\xa5\x69\xe2\x95\x46\xe4\x67\x5e\x94\x54\xae\x24\x15\x7d\xbf\xb8\xf5\xbe\x58\x8d\x30\xf0\x1e\x90\xe2\xe5\xed\xb9\x3a\x9d\xf8\x88\xb6\xe1\x29\xc3\x9d\xc8\xb1\x09\x23\x86\x5d\x4f\x26\x86\xdd\x88\x1e\x9b\xf7\xb5\xd8\x7c\x14\xd9\x00\xd9\x7c\x75\x7c\xa9\x9c\x37\x72\x12\xcf\x70\xa1\xdb\xcc\x57\xf3\xd3\xc6\xa0\x32\x82\x55\xa7\x2a\xf2\x52\xb7\x1a\x42\x37\xdb\x0e\x94\xfc\xc1\xee\x3a\x38\x66\xb5\x57\x0d\xd6\xf8\xb7\x1a\xf2\x07\xc6\x25\x9f\x37\x45\x09\xa9\xad\x36\xd1\x6e\x36\xfd\xf1\xcc\xc1\xa9\xbb\xd7\x87\x8f\x3b\xaf\x66\x60\x33\xd8\x87\xa6\xf7\x22\x13\xf6\x7b\x34\xbf\xec\xdc\x83\x29\x5f\xb5\x16\xa1\x38\x34\xdd\x6c\x94\x2d\x34\x37\x6b\xea\x78\x45\x1b\x3d\x1b\xc5\x8b\x83\x6b\x7f\x1a\x94\xc9\xef\xcc\xa7\xea\xcd\xfc\xa8\x17\xce\x01\xf2\x54\x31\xb8\xf0\x64\xfd\x07\xfc\x78\x96\x3b\xc6\x0a\xfc\x71\x68\x06\xcb\xfa\x69\x58\xda\x35\xfe\x38\xb5\x9d\xca\x1e\x1c\xac\xdd\x47\x8e\xab\x5e\xb0\x3b\xaa\x1e\x61\xc4\x1b\xe4\xd3\x81\x0a\x3b\xa2\xbc\xdd\x81\x8e\xd3\xd2\xee\x50\xa3\xdb\xb7\xdb\xd5\x91\x87\xa2\x6c\x67\xdd\xd1\xbd\xfd\x93\x89\x70\x37\x7b\x06\xa4\xf5\x7b\xf4\x87\xe5\x75\x41\xb9\xde\x6d\x92\x6f\xc4\xa5\x1a\x2d\x8b\xdf\x67\xdc\x97\xef\x67\x22\xe2\xdf\x52\xf8\x77\x98\xf1\x52\xae\xd5\x9d\xf4\x85\xb1\x2e\x99\x45\xd5\xf9\xc2\xd7\xc0\x72\x26\xe0\x46\xc5\xd5\xcf\x3d\x56\xf5\xaf\x95\xfc\x9f\x0b\xbd\x81\x14\x0c\x15\x93\xff\x04\x00\x00\xff\xff\x27\xe4\x4c\xed\xb9\x29\x00\x00") func deployKubernetes118DirectPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -139,12 +139,12 @@ func deployKubernetes118DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.18/direct/pmem-csi.yaml", size: 10274, mode: os.FileMode(420), modTime: time.Unix(1599471426, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.18/direct/pmem-csi.yaml", size: 10681, mode: os.FileMode(420), modTime: time.Unix(1599822738, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes118LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xd3\x7d\xd8\x05\x2a\x2b\xde\xa2\xc0\x42\x80\x1f\x32\x13\xef\xd4\xe8\xc4\x31\x92\xec\xbe\x06\x34\x75\x2c\xb3\xa6\x48\x2e\x79\xa4\x89\xb6\xe8\x7f\x2f\xa8\x8b\xad\x9b\xe5\xcb\xe6\xd2\xcd\x3c\x8c\x23\xf2\xf0\x7c\x3c\xd7\xef\x58\xf9\x2b\xf9\x02\x12\x0c\x45\x08\xc9\x37\x8e\x1b\x72\x15\xd3\x2d\x90\x6d\x62\x51\xc5\xfc\x77\xb8\xfa\x1b\x09\x15\x91\x0a\x09\x84\x1c\xff\x32\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x01\x49\x27\xa3\x2d\x97\x61\x40\x1e\xc1\xa4\x9c\xc1\x0d\x63\x2a\x91\x38\x8a\x01\x69\x48\x91\x06\x23\x42\x04\x5d\x81\xb0\xee\x13\x21\x3a\x86\xd8\x63\x96\x8f\xb9\x44\x10\x63\xa6\x62\x3f\x04\x2d\x54\x16\x83\xc4\x80\x88\x34\xf6\xb4\x51\x61\xc2\x90\x2b\x39\x22\x44\xd2\x18\x82\x9d\x94\xc7\x94\x44\xa3\x84\x00\x53\xae\x59\x4d\x19\x04\x24\x84\x35\x4d\x04\x8e\x3c\xcf\x6b\xc0\x33\x2b\xca\xc6\x34\xc1\x8d\x32\xfc\x77\xea\xce\x1c\x6f\x7f\xb2\x63\xae\xfc\x1d\xf0\x07\x25\xe0\xad\xe0\xc2\x0b\x82\x91\x54\xb8\x4d\x29\x77\x90\xc0\x78\x6c\x1d\xf5\x83\x37\x89\x00\x1b\x8c\x3c\x42\x35\xff\x62\x54\xa2\x73\x10\x1e\xb9\xba\x1a\x11\x62\xc0\xaa\xc4\x30\x28\x9f\x81\x0c\xb5\xe2\x12\xed\x88\x90\x14\xcc\xaa\x7c\x1c\x01\xe6\xff\x7f\xa3\xc8\x36\xf9\x27\xc1\x6d\xf1\x28\x04\x01\x08\xf9\xc7\x44\x87\xb4\xfc\xc8\x0c\xb8\x8f\x1d\x9d\x4c\x29\x13\x72\x59\x37\x59\x17\x84\x00\x6a\xe1\xad\x10\x58\x54\x86\x46\x70\x50\x39\xb3\xbc\xdc\xc2\xa8\xa6\x8c\x23\x3f\x04\x65\x07\x60\x8f\xa9\x54\xda\x82\xa2\x77\xeb\x25\xd4\x93\x5c\xa1\x55\xd8\xa3\xb8\x23\x4a\xb5\xb6\x5d\x61\x03\x5a\x70\x46\x2d\xf4\x79\xf2\x82\x68\xfe\x2c\x12\x8b\x60\xde\x3d\xa8\x4d\x22\x25\x98\xb3\x42\x58\xbb\x6b\x59\x04\x89\xa9\x12\x49\x7c\x99\xf7\xce\x72\x54\x4b\x21\x13\x94\xc7\xa7\x6b\x2d\x03\xe5\xec\x40\xad\xa2\x54\x50\x7b\x30\x5b\x5a\xea\x4e\x2b\x01\x29\x74\xf2\xff\xcc\x50\xef\xde\x45\x52\x6d\x37\x0a\xc7\xc7\x2e\x55\xba\xac\xdc\x3e\x74\xab\xd7\xd2\xe1\x4a\x7f\xf7\xc2\x47\x54\x1d\xaf\x21\x52\x85\xaf\xe9\x94\x3f\x70\xdc\x69\xe6\xa0\x88\x94\x6d\xe2\x23\x96\xa8\xa9\xb9\xac\x23\x7e\xe2\x32\xe4\x32\x7a\xb3\x3e\x6e\x79\xb3\x7c\x28\x01\x03\x8d\x51\x09\x78\x80\xb5\xd3\x5c\xd9\x6c\xe0\x22\x23\x42\x6a\x9d\xfd\xf4\x9e\x6c\x93\xd5\xbf\x81\x61\x5e\xbe\x7a\x39\xcd\x7b\xb0\x91\x5a\xfd\xfe\x00\x17\x5c\x6a\xe9\x7a\xd7\x39\xab\x5f\xbc\xbd\xcd\xdb\x04\xf5\xa3\x98\xa9\xd5\xc0\x9c\x22\xad\x0c\xee\x88\x83\xc1\x80\x4c\xae\xaf\xaf\xaf\x73\x04\x48\x4d\x04\xb8\x6c\x3c\xb5\x20\x80\xa1\x32\x05\x46\xaa\xf5\x21\x95\xe7\x5e\xe0\xa3\xec\x14\x03\x1a\xce\xec\xf9\x46\x9a\xf4\x1a\x69\xf2\xa6\x46\x22\x04\x33\x0d\x01\x59\xa8\x10\x9c\xca\x8e\xd5\x1c\xa9\xdb\x27\xef\x23\x52\x84\x75\x22\x1e\xe1\xc3\x06\xa0\xca\x82\x15\xaf\x0c\xc8\xa4\x63\xa0\xd8\x35\x86\xaf\x35\x4c\x47\x6c\x76\x3e\x68\x5b\xc4\xd0\x62\x00\x3b\x42\xac\x05\x45\x28\x21\xd5\xac\xe5\x7e\x44\x03\xdd\x51\x7c\xe7\x23\x3c\x28\xf5\x0d\x56\x1b\xa5\xb6\x01\xe1\x91\x54\x06\xf2\xad\x95\x51\x73\x24\xeb\x35\x97\x1c\xb3\x3d\x36\xd7\xf2\x6f\x3a\x4f\x9d\x0b\x7e\x4b\xb8\x81\xf0\x36\x31\x5c\x46\x8f\x6c\x03\x61\x22\xb8\x8c\xe6\xf9\xc1\xe5\xe3\xd9\x0b\xb0\xc4\x61\xaa\x4b\x16\x67\x3e\x96\x4e\x7b\x02\x13\xdb\xe6\xb2\x57\xf8\x70\xf6\xa2\x0d\x58\x17\x89\xad\x75\xb7\x63\x0b\x59\xd0\x77\xc1\x1e\xe3\x55\x3f\x4a\x83\xa1\x2e\x4c\xc8\x42\xe1\x5c\x76\xd6\x53\x2a\x12\xe8\xa8\xca\xa9\x90\x54\x57\x7d\x8f\xd7\x54\x58\xa8\x56\x9c\x66\xca\x25\x98\xdd\x11\x6e\xce\x8c\x63\x2a\xc3\xfd\x99\x1e\xf1\x13\x6b\x7c\xa1\x18\x15\xfe\x8a\x4b\x7f\xe7\xf5\xd0\xf0\xb4\x06\xda\x23\x5e\x3a\xfd\x7b\xfd\xd7\x58\x85\x30\xed\xb9\x9e\x47\xbc\x6a\x5c\x9e\x26\x92\xbf\x04\xbe\xef\x33\xcb\xfd\x66\x28\x8d\xad\x62\xdb\xba\x8c\x81\x88\x5b\x34\xd9\xac\x92\x45\xa6\x03\xdf\xbf\x1e\xe7\xff\x82\x7d\xdd\xae\xd4\x17\x95\xed\x6b\x3e\x62\x4c\x83\x7d\xc9\x2a\xd7\x9d\x4f\x79\x38\xfd\xee\xfb\x7f\xfd\xf2\x69\xf6\xbc\xb8\xbf\x9d\x3d\x2f\x6e\xee\x66\x3f\xd4\xf7\x30\xfa\x33\x17\x30\xf5\x19\x18\xb4\x3e\xa3\x63\x66\xb0\xb1\x0e\x06\xeb\x3b\x50\xd8\xf6\x96\x2d\x64\xed\x1d\x5b\xc8\xea\x3b\x0a\x43\xba\xe2\x31\xfd\xee\xfb\xe5\xdd\xec\xee\xf9\xf3\xe3\xfc\xf9\xf6\x61\xfe\xeb\xec\xa1\x05\x09\x64\x5a\x77\x4d\x51\x8c\x9a\xf8\x47\xad\xf8\xf8\xd9\xa8\xb8\x19\x22\x6b\x0e\x22\x2c\x59\x45\xfd\xa7\xd5\x7e\x9a\x8b\xb9\xd0\x92\xe2\x26\xc8\xd3\x6f\xec\xac\xe7\xea\x49\x07\xcd\xd3\xec\xe1\x6e\xbe\xb8\x79\x9a\xdf\x2f\x9e\xbf\xde\x7f\x79\x5e\xde\x3c\xfd\xb3\x8d\x29\x20\x3e\xc6\xda\x47\x30\x71\xf9\x9d\x86\x27\x54\xd4\x39\xaa\xcf\x16\xdd\xa3\xba\x39\xb5\xdb\xc3\x63\x1a\x41\x40\xf2\x85\x76\xe0\x06\x8c\x4a\x6a\xb2\xe6\xde\x65\x22\xc4\x52\x09\xce\xb2\x80\xcc\xd7\x0b\x85\x4b\x03\x16\xe4\xde\xa1\xb5\xf2\xdf\x8a\x7f\x0b\x2c\x31\x1c\xb3\xcf\x6e\x2e\x7a\xc1\x66\xe5\xa1\xe1\xbd\x14\xd9\x83\x52\x79\xb4\xd8\xcc\x22\xc4\x01\x41\x93\xec\xed\x57\x33\xc6\x1d\x58\xeb\xc0\xe4\xd6\x1e\xb4\x54\x31\x83\xdc\x39\x6a\x66\xeb\x81\x11\xbb\x27\xa5\x7c\x1e\x77\x35\x34\xc5\x15\xaa\x6c\xca\x43\xf8\x90\xa4\xe5\x1d\x39\x2d\x92\x88\x4b\xcf\x25\x27\xa0\x17\x72\x73\x40\x16\x63\xdd\x91\xc5\x58\xd7\x24\x3c\x42\x4d\xd4\x40\xdd\x2e\x20\xb9\xb7\x68\x18\xba\xa2\x3a\x3d\xa5\x44\x78\x6b\xa0\x98\x18\xf0\x22\x8a\x60\xa7\x4f\x4a\x2b\xa1\xa2\x6c\xda\x30\xb4\xdb\x67\x5d\x69\x40\x0f\x0f\x6e\x40\x1e\x83\x4a\x70\xfa\x8f\x4e\x30\xfd\x96\xd0\xcc\x8d\x08\xdb\x9f\x6c\x85\xa8\x46\xa4\x83\xf4\x47\x57\x8d\x2e\x8a\xaa\x3e\x62\xfe\x6a\xe1\x75\x52\xa4\x9c\xe1\xef\x83\x78\x4c\x22\x6f\xec\x42\x49\x87\xa6\x85\x21\x5f\xfa\xc5\x82\x29\xf8\xf4\xee\xa4\xfa\x84\x31\xc4\x51\xf6\xf7\xa8\x75\x2c\x88\x35\x66\xb7\xdc\x04\x44\x26\x42\xb4\x13\xf5\x00\x7a\x6f\x30\x0b\x2c\x30\x03\x0d\x13\x17\x4f\x5a\xd8\x76\xc2\xc5\xaa\xed\x81\xf4\x9f\xff\x8e\xfa\x53\x60\x98\xb8\xde\x52\x88\x95\x7c\x43\xda\xea\xaa\xf7\x30\x61\x3d\x93\xa2\x96\x07\x5e\x02\xed\x8f\x30\xcf\x9a\xda\x77\xe4\x9c\xaf\x4e\x9c\x42\x70\x19\x70\x47\x25\x8d\xc0\x4c\x45\x1a\x1f\x63\x55\x8d\x7b\x0f\xf0\xa9\x4e\x85\x3c\x89\xf4\xec\x72\xae\x45\xb5\x4a\xa9\xe5\xfd\xed\xf3\x7c\xf9\x43\xce\xb8\x26\x27\x30\xb4\x9e\x64\xee\xb2\xb5\x77\x61\x5a\xd6\x0d\x84\xae\xd8\x4d\xfd\x94\x1a\x5f\xf0\x95\x7f\x8c\x71\x9d\xc7\xcf\x3c\xe2\xb9\xdb\x2e\xc1\x30\x90\x48\x23\x98\x4e\x6a\xd7\xfc\x13\xb0\xb7\x9a\x83\xdf\x01\x0a\x52\x4c\xec\x58\xab\xb0\xa6\xed\xd5\xc8\xdf\x2b\x12\xd2\xff\x13\x1e\xa9\x0d\x4f\xb9\x80\x08\xc2\x56\x63\x6d\xb4\xd6\xeb\xb7\xa7\x94\x55\xf2\x6c\x93\x15\x08\x40\xbf\x68\xb3\x36\xff\xdd\x48\x40\xc8\xbf\x44\x6d\xd2\x89\x42\xde\x28\x4d\xa3\x5c\x69\x40\x3e\xf1\x90\x1b\xc8\xab\x31\x15\x1d\xe2\x91\xef\xcf\xcb\xc8\x00\xcb\xec\x00\x29\xde\xfd\x5d\xaa\xd3\x89\x0f\x68\xeb\x67\xd2\x2e\x97\x86\x58\x74\x08\x69\x47\x26\x84\x74\x40\x8f\xcd\xba\x5a\x6c\x36\x88\xac\x87\xb9\x1d\xa5\xe8\x95\xf1\x06\x72\xe8\x02\x13\xba\xc3\xf2\x3a\x7b\x1e\xd5\x2f\x3d\x58\xf5\x90\x22\x2e\x75\xa3\x54\xb7\xa3\xed\x40\x31\xee\xed\x7b\xbd\xa3\x44\x73\x57\x6f\x75\x7e\xad\x11\xb4\x67\x6a\xc8\xe3\xa6\x48\xfe\xdd\xad\x4d\x90\x4e\xc6\x3f\x5e\x38\x3f\xb4\xcf\xfa\xf0\xd9\xe1\x68\x04\xd6\x9d\x7d\x68\x42\x2d\x22\x61\x7f\x46\xfd\xab\xb8\x3d\x98\xf2\x65\x5d\xe1\x8a\x43\xa3\xc2\x46\xd9\x42\x73\xbd\xa6\x0e\x57\xb4\xc1\xdc\x28\xbe\x8c\xbe\xcd\xb3\x41\x99\xec\xde\x7c\xae\xde\xed\x0e\x5a\xe1\x12\x20\xcf\x15\xb7\xf2\xcf\xd6\x7f\xc0\x8e\x17\x99\x63\xa8\xc0\x9f\x86\xa6\xb7\xac\x9f\x87\xa5\x59\xe3\x4f\x53\xdb\xaa\xec\xde\xc1\xda\x7d\xe2\xec\x97\x0b\xb6\xe7\xbe\x13\x2e\xf1\x0a\xf1\x74\xa0\xc2\x0e\x28\x6f\x76\xa0\xd3\xb4\x34\x3b\xd4\xe0\xf1\xcd\x76\x75\x62\x52\x94\xed\xac\x3d\x07\x37\x5f\xba\xfb\xe9\x64\x05\x48\x77\x2f\x63\x1f\xe7\xb7\x05\x59\x7a\x9b\xb1\xb8\xe6\x94\x6a\xdc\x2b\x5e\xef\x3f\x94\xaf\x0e\x02\x92\x7f\x81\x9e\xbf\x11\x0b\xe7\x72\xad\xee\x65\x5e\x15\x77\xf5\xb2\x28\x39\x5f\xf9\x1a\x58\xc6\x04\xdc\xa9\xb0\xfa\x6b\x81\xe5\xee\x8f\x5d\xf2\x5f\x67\x7a\x03\x31\x18\x2a\x46\xff\x0b\x00\x00\xff\xff\xf6\xb2\x6e\xe3\xef\x27\x00\x00") +var _deployKubernetes118LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xa7\xfb\xb0\x0b\x54\x56\xbc\x45\x81\x85\x00\x3f\x64\x27\xde\xa9\xd1\x49\x62\x24\xd9\x7d\x0d\x18\xea\x58\x66\x4d\x91\x5c\xf2\x48\x13\x6d\xd1\xff\x5e\x90\x92\x6c\xdd\xac\xd8\x6e\x2e\x68\x3d\x0f\xa3\x48\x24\xcf\x77\xee\xdf\xb1\xfc\x67\xf2\x05\x24\x18\x8a\x10\x93\x6f\x1c\x37\xe4\x53\x4a\xb7\x40\xb6\x99\x45\x95\xf2\x3f\xe0\xd3\x5f\x48\xac\x88\x54\x48\x20\xe6\xf8\xa7\xc9\x84\x6a\xfe\x1b\x18\xcb\x95\x8c\x48\x3e\x9b\x6c\xb9\x8c\x23\x72\x0f\x26\xe7\x0c\x2e\x19\x53\x99\xc4\x49\x0a\x48\x63\x8a\x34\x9a\x10\x22\xe8\x13\x08\xeb\xae\x08\xd1\x29\xa4\x01\xb3\x7c\xca\x25\x82\x98\x32\x95\x86\x31\x68\xa1\x8a\x14\x24\x46\x44\xe4\x69\xa0\x8d\x8a\x33\x86\x5c\xc9\x09\x21\x92\xa6\x10\xed\x76\x05\x4c\x49\x34\x4a\x08\x30\xd5\x33\xab\x29\x83\x88\xc4\xb0\xa6\x99\xc0\x49\x10\x04\x2d\x78\xe6\x89\xb2\x29\xcd\x70\xa3\x0c\xff\x83\xba\x33\xa7\xdb\x9f\xec\x94\xab\x70\x07\xfc\x4e\x09\x78\x2b\xb8\xf0\x8c\x60\x24\x15\x6e\x51\xce\x1d\x24\x30\x01\x5b\x27\xc3\xe0\x4d\x26\xc0\x46\x93\x80\x50\xcd\xbf\x18\x95\x69\x0f\x22\x20\x9f\x3e\x4d\x08\x31\x60\x55\x66\x18\x54\xf7\x40\xc6\x5a\x71\x89\x76\x42\x48\x0e\xe6\xa9\xba\x9d\x00\xfa\xff\xbf\x51\x64\x1b\x7f\x25\xb8\x2d\x6f\xc5\x20\x00\xc1\x5f\x66\x3a\xa6\xd5\x25\x33\xe0\x2e\x7b\x32\x99\x52\x26\xe6\xb2\x69\xb2\x3e\x08\x01\xd4\xc2\x5b\x21\xb0\xa8\x0c\x4d\xe0\xa0\x70\x66\x79\xb5\x84\x51\x4d\x19\x47\x7e\x08\xca\x0e\xc0\x1e\x53\x25\xb4\x03\x45\xef\x9e\x57\x50\x8f\x72\x85\x56\xf1\x80\xe0\xde\x56\xaa\xb5\xed\x6f\x36\xa0\x05\x67\xd4\xc2\x90\x27\xcf\x88\xe6\xcf\x22\xb3\x08\xe6\xdd\x83\xda\x64\x52\x82\x39\x29\x84\xb5\x53\xcb\x22\x48\xcc\x95\xc8\xd2\xf3\xbc\x77\x92\xa3\x3a\x02\x99\xa0\x3c\x3d\x5e\x6a\x15\x28\x27\x07\x6a\x1d\xa5\x82\xda\x83\xd9\xd2\x11\x77\x5c\x09\xc8\xa1\x97\xff\x27\x86\x7a\x5f\x17\x49\xb5\xdd\x28\x9c\xbe\xa4\x54\xe5\xb2\x6a\xf9\x98\x56\xaf\x25\xc3\x95\xfe\xbe\xc2\x2f\x88\x7a\xb9\x86\x48\x15\xbf\xa6\x53\xfe\x8b\xe3\x8e\x33\x07\x45\xa4\x6c\x93\xbe\x60\x89\x86\x98\xf3\x3a\xe2\xcf\x5c\xc6\x5c\x26\x6f\xd6\xc7\x2d\x6f\x97\x0f\x25\x60\xa4\x31\x2a\x01\x77\xb0\x76\x92\x6b\x9b\x8d\x28\x32\x21\xa4\xd1\xd9\x8f\xef\xc9\x36\x7b\xfa\x27\x30\xf4\xe5\x6b\x90\xd3\xbc\x07\x1b\x69\xd4\xef\x0f\x70\xc1\xb9\x96\x6e\x76\x9d\x93\xfa\xc5\xdb\xdb\xbc\x4b\x50\x3f\x8a\x99\x5a\x0d\xcc\x09\xd2\xca\xe0\x8e\x38\x18\x8c\xc8\xec\xe2\xe2\xe2\xc2\x23\x40\x6a\x12\xc0\x55\xeb\xae\x05\x01\x0c\x95\x29\x31\x52\xad\x0f\x89\x3c\x55\x81\x8f\xb2\x53\x0a\x68\x38\xb3\xa7\x1b\x69\x36\x68\xa4\xd9\x9b\x1a\x89\x10\x2c\x34\x44\xe4\x46\xc5\xe0\x44\xf6\xac\xe6\x48\xdd\x3e\x79\xef\x91\x22\xac\x33\x71\x0f\x1f\x36\x00\xd5\x16\xac\x79\x65\x44\x66\x3d\x03\xa5\xae\x31\x7c\x6d\x60\x7a\xc1\x66\xa7\x83\xb6\x65\x0c\xdd\x8c\x60\x47\x48\xb5\xa0\x08\x15\xa4\x86\xb5\x3c\x1e\x29\x15\xfa\x52\xb3\x83\x38\x88\xc2\x32\x43\x9d\x83\xdc\xd1\x94\x4b\x30\xb6\x5a\x2d\x5a\xea\xbd\xa8\xe0\xe9\x2a\x1e\xdc\xf5\x0d\x9e\x36\x4a\x6d\x23\xc2\x13\xa9\x0c\xf8\xa5\xb5\x57\x3c\x92\xf5\x9a\x4b\x8e\xc5\x1e\x9b\xe3\x0c\x97\xbd\xbb\xce\x87\xbf\x67\xdc\x40\x7c\x95\x19\x2e\x93\x7b\xb6\x81\x38\x13\x5c\x26\x4b\x7f\x70\x75\x7b\xf1\x0c\x2c\x73\x98\x9a\x3b\xcb\x33\xef\x2b\xaf\x3f\x80\x49\x6d\xfb\x71\x50\x06\xc1\xe2\x59\x1b\xb0\xb6\x6d\xe6\x7a\xc5\x16\x8a\x68\x48\xc1\x01\xe3\xd5\x1f\xa5\xc1\x50\x17\x67\xe4\x46\xe1\x52\xf6\x9e\xe7\x54\x64\xd0\x13\xe5\xb9\x94\x54\x9f\x86\x6e\xaf\xa9\xb0\x50\x3f\xd9\x7b\xb9\x3e\xc2\x0d\xaa\x69\x4a\x65\xbc\x3f\x33\x20\x61\x66\x4d\x28\x14\xa3\x22\x7c\xe2\x32\xdc\x79\x3d\x36\x3c\x6f\x80\x0e\x48\x90\xcf\xff\xda\xfc\x33\x55\x31\xcc\x07\xd4\x0b\x48\x50\xcf\xdb\xf3\x4c\xf2\xe7\x28\x0c\x43\x66\x79\xd8\x0e\xa5\xa9\x55\x6c\xdb\xdc\x63\x20\xe1\x16\x4d\xb1\xa8\xf7\x22\xd3\x51\x18\x5e\x4c\xfd\xbf\x68\x5f\xf8\xab\xf5\xce\x67\x3c\x9e\x7f\xf7\xfd\x3f\x7e\xfd\x79\xf1\x78\x73\x7b\xb5\x78\xbc\xb9\xbc\x5e\xfc\xd0\x5c\xc3\xe8\x2f\x5c\xc0\x3c\x64\x60\xd0\x86\x8c\x4e\x99\xc1\xd6\x73\x30\xd8\x5c\x81\xc2\x76\x97\x6c\xa1\xe8\xae\xd8\x42\xd1\x5c\x51\x1a\xca\x55\x97\xf9\x77\xdf\xaf\xae\x17\xd7\x8f\x9f\xef\x97\x8f\x57\x77\xcb\xdf\x16\x77\x7d\x48\x55\x45\xff\xea\x47\xab\x79\xb4\x2f\xd5\xee\x03\x32\x6f\xba\xa6\xac\x66\x6d\xfd\x26\x9d\xf8\xf8\xc5\xa8\xb4\x1d\x22\x6b\x0e\x22\xae\x68\x49\xf3\xd3\xe9\x5f\xed\x87\x7e\xd3\x8a\xe2\x26\xf2\xe9\x37\x75\xd6\x75\x05\xa9\x87\xe6\x61\x71\x77\xbd\xbc\xb9\x7c\x58\xde\xde\x3c\x7e\xbd\xfd\xf2\xb8\xba\x7c\xf8\x7b\x17\x53\x44\x42\x4c\x75\x88\x60\xd2\xea\x4b\x91\x40\xa8\xa4\x77\xd4\x90\xad\xfa\x47\xf5\x73\x6a\xb7\x86\xa7\x34\x81\x88\xf8\x07\xdd\xc0\x8d\x18\x95\xd4\x14\xed\xb5\xab\x4c\x88\x95\x12\x9c\x15\x11\x59\xae\x6f\x14\xae\x0c\x58\x90\x7b\x87\x37\xfa\x47\x27\xfe\x77\x4d\xb6\xd6\x61\x97\x5e\xab\x4e\xcf\x6d\x9e\xb4\xef\xdf\xe5\xc7\x02\xcb\x0c\xc7\xe2\xb3\x1b\xcf\x9e\xb1\x5d\xbf\x68\x7c\x2b\x45\x71\xa7\x94\x8f\x49\x5b\x58\x84\x34\x22\x68\xb2\xbd\x17\x1a\x26\xbd\x06\x6b\x9d\x4a\xde\x67\xa3\xf6\x2e\x47\xa1\x6b\xc7\x10\x5b\x1a\xa4\xee\x4e\xb5\xdf\x47\x77\x0f\x7e\x9d\x93\x3e\x51\x0e\xed\xb4\xbc\xb7\x4f\x8b\x2c\xe1\x32\x70\x29\x0e\x18\xc4\xdc\x1c\xd8\x8b\xa9\xee\xed\xc5\x54\x37\x76\x04\x84\x9a\xa4\x85\xba\x5b\x86\xbc\xcf\x69\x1c\xbb\xd2\x3c\x3f\xa6\xd0\x04\x6b\xa0\x98\x19\x08\x12\x8a\x60\xe7\x0f\x4a\x2b\xa1\x92\x62\xde\x32\xb4\x5b\x67\x9d\xef\x30\xc0\x83\x0b\x90\xa7\xa0\x32\x9c\xff\x2d\x6d\xdd\xae\x9c\xbe\x03\xe5\x93\x7c\xd6\x8d\xda\xdf\x33\x5a\xb8\x61\x66\xfb\x93\xad\x41\x37\x28\x7f\x94\xff\xe8\xca\xde\x59\xe1\x3b\x34\x42\x9c\x14\xc7\xb3\x37\x8f\xe3\xa3\x42\xf2\x84\xc0\x3a\x88\xc7\x64\xf2\xd2\xde\x28\xe9\xd0\x74\x30\xf8\x47\xbf\x5a\x30\xe5\xfc\xb0\x3b\xa9\x39\x51\x8d\x71\xb2\xbd\x1e\x8d\x06\x0b\xa9\xc6\xe2\x8a\x9b\x88\xc8\x4c\x88\x6e\x5d\x39\x80\x3e\x18\x4d\x37\x0b\xcc\x40\xcb\xc4\xe5\x9d\x0e\xb6\xdd\xe6\xf2\xa9\x1d\x80\xf4\xaf\x7f\x77\x00\xd5\xb9\x36\x4e\xd4\xaf\x28\xa4\x4a\xbe\x21\x4d\x77\xcd\x66\x9c\xa0\x9f\x48\xc9\xab\x03\xcf\x81\xf6\xa1\x4c\xbb\x81\xfb\x1d\x39\xf6\xab\x13\xc5\x18\x5c\x0a\x5d\x53\x49\x13\x30\x73\x91\xa7\x2f\xb1\xc8\x96\xde\x23\xfc\xb1\x57\xcb\x8f\x22\x81\xbb\xa4\xed\x50\xcb\x6a\xd7\xea\xf6\xea\x71\xb9\xfa\xc1\x33\xcc\xd9\x11\x8c\x74\xa0\x1a\xf4\xd9\xe9\xbb\x30\x4f\xeb\x26\x68\x57\x2d\xe7\x61\x4e\x4d\x28\xf8\x53\x78\x04\x03\x3d\x8d\xaf\x3a\x6d\x57\x60\x18\x48\xa4\x09\xcc\x67\x6d\x35\xff\xd7\xd9\x6c\x23\x00\xde\x01\x0a\x52\xcc\xec\x54\xab\xb8\x21\xed\xd5\xc8\xf0\x2b\x12\xf4\xff\x2b\x5e\xad\x0d\xcf\xb9\x80\x04\xe2\x4e\xff\x6f\x31\x80\x8b\xb7\xa7\xd8\x75\x8a\x6e\xb3\x27\x10\x80\x61\xc9\x06\xac\xff\xdb\x48\x40\xf0\xdf\x6d\xb7\x59\x4f\xb9\xdf\x28\x4d\x13\x2f\x34\x22\x3f\xf3\x98\x1b\xf0\x35\x9f\x8a\xbe\x5d\xdc\x7a\x5f\xac\x46\x58\x77\x0f\x48\xf9\x4a\xf6\x5c\x99\x6e\xfb\x88\xb4\xe1\xc9\xc2\x65\xe4\xd8\x54\x11\x43\xde\xdb\x13\x43\x3e\x22\xc7\x16\x7d\x29\xb6\x18\x45\x36\x40\x30\x5f\x1c\x59\x6a\xe3\x8d\x64\xe2\x19\x26\x74\x87\xf9\x6a\x7e\xda\xe8\x53\x79\xb0\xee\x54\x65\x5c\xea\x56\x43\xe8\x46\xdb\x81\x92\x3f\xd8\x5d\x07\x47\xab\xf6\xaa\xc1\x1a\xff\x5a\x83\xfd\xc0\x88\xe4\xe3\xa6\x2c\x21\x3b\xad\x4d\x94\xcf\xa6\x3f\x9e\x39\x2c\x75\xcf\xfa\xf0\x11\xe7\xc5\x08\x6c\x3a\xfb\xd0\xc4\x5e\x46\xc2\xfe\x8c\xe6\x17\x9c\x7b\x30\xd5\x3b\xd4\xd2\x15\x87\x26\x9a\x8d\xb2\xa5\xe4\x66\x4d\x1d\xaf\x68\xa3\xb9\x51\xbe\x23\xb8\xf2\xd9\xa0\x4c\x71\x6b\x3e\xd7\xaf\xdc\x47\xad\x70\x0e\x90\xc7\x9a\xc1\x85\x27\xcb\x3f\x60\xc7\xb3\xcc\x31\x56\xe0\x8f\x43\x33\x58\xd6\x4f\xc3\xd2\xae\xf1\xc7\x89\xed\x54\xf6\xe0\x60\xed\x3e\x72\x44\xf5\x1b\xbb\xe3\xe9\x11\x4a\xbc\x42\x3c\x1d\xa8\xb0\x23\xc2\xdb\x1d\xe8\x38\x29\xed\x0e\x35\x7a\x7c\xbb\x5d\x1d\x99\x14\x55\x3b\xeb\x8e\xeb\xed\xdf\x42\x84\xf9\xec\x09\x90\xee\xde\x91\xdf\x2f\xaf\x4a\xca\xf5\x36\xd3\x7b\xc3\x29\xf5\x50\x59\xfe\xea\xe2\xae\x7a\x21\x13\x11\xff\x5a\xc2\xbf\xa8\x8c\x97\x72\xad\x6e\xa5\xaf\x8a\xbb\x7a\x59\x96\x9c\xaf\x7c\x0d\xac\x60\x02\xae\x55\x5c\xff\x88\x63\xb5\xfb\x0d\x92\xff\x73\xa1\x37\x90\x82\xa1\x62\xf2\x9f\x00\x00\x00\xff\xff\xb2\xb2\xba\xec\x86\x29\x00\x00") func deployKubernetes118LvmPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -159,12 +159,12 @@ func deployKubernetes118LvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.18/lvm/pmem-csi.yaml", size: 10223, mode: os.FileMode(420), modTime: time.Unix(1599471426, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.18/lvm/pmem-csi.yaml", size: 10630, mode: os.FileMode(420), modTime: time.Unix(1599822739, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119AlphaDirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5b\x6f\xe3\xb8\xf5\x7f\xf7\xa7\xe0\x3f\xff\x7d\xd8\x05\x2a\x2b\xd9\xa2\xc0\x42\x80\x1f\x32\x89\x77\x6a\x74\x92\x18\x49\x76\x5f\x03\x86\x3a\x96\x59\x53\x24\x97\x3c\xf2\x44\x5b\xf4\xbb\x17\xa4\x24\x5b\x37\xcb\x97\x71\x26\x6d\xe6\x61\x6c\xde\xce\x8f\xe7\xfa\x3b\x92\xff\x9f\x7c\x06\x09\x86\x22\xc4\xe4\x2b\xc7\x25\xb9\x48\xe9\x0a\xc8\x2a\xb3\xa8\x52\xfe\x27\x5c\xfc\x85\xc4\x8a\x48\x85\x04\x62\x8e\xff\x37\x1a\x51\xcd\x7f\x07\x63\xb9\x92\x11\x59\x5f\x8d\x56\x5c\xc6\x11\x79\x02\xb3\xe6\x0c\xae\x19\x53\x99\xc4\x51\x0a\x48\x63\x8a\x34\x1a\x11\x22\xe8\x2b\x08\xeb\x3e\x11\xa2\x53\x48\x03\x66\xf9\x98\x4b\x04\x31\x66\x2a\x0d\x63\xd0\x42\xe5\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x22\x44\xd2\x14\xa2\xcd\xc6\x80\x29\x89\x46\x09\x01\xa6\x9c\xb3\x9a\x32\x88\x48\x0c\x0b\x9a\x09\x1c\x05\x41\xd0\x40\x68\x5e\x29\x1b\xd3\x0c\x97\xca\xf0\x3f\xa9\x3b\x73\xbc\xfa\xc5\x8e\xb9\x0a\x37\xd8\x1f\x95\x80\x77\x44\x0c\x6f\x08\x46\x52\xe1\x16\xad\xb9\x43\x05\x26\x60\x8b\xa4\x1f\xbf\xc9\x04\xd8\x68\x14\x10\xaa\xf9\x67\xa3\x32\xed\x71\x04\xe4\xe2\x62\x44\x88\x01\xab\x32\xc3\xa0\x1c\x03\x19\x6b\xc5\x25\xda\x11\x21\x6b\x30\xaf\xe5\x70\x02\xe8\xff\xff\x4a\x91\x2d\xfd\x27\xc1\x6d\x31\x14\x83\x00\x04\xff\x31\xd3\x31\x2d\x3f\x32\x03\xee\x63\x47\x26\x53\xca\xc4\x5c\xd6\xb5\xd6\x05\x21\x80\x5a\x78\x2f\x04\x16\x95\xa1\x09\xec\x14\xce\x2c\x2f\x97\x30\xaa\x29\xe3\xc8\x77\x41\xd9\x00\xd8\x62\x2a\x85\xb6\xa0\xe8\xcd\x7c\x09\xf5\x20\x53\x68\x15\xf7\x08\xee\x6c\xa5\x5a\xdb\xee\x66\x03\x5a\x70\x46\x2d\xf4\x59\xf2\x04\x87\xbe\x11\x99\x45\x30\x1f\xe1\xd7\x26\x93\x12\xcc\x51\x5e\xac\xdd\xcd\x2c\x82\xc4\xb5\x12\x59\x7a\x9a\x01\x8f\xb2\x55\x4b\x20\x13\x94\xa7\x87\x4b\x2d\x7d\xe5\x68\x5f\xad\x1c\x55\x50\xbb\x33\x60\x5a\xe2\x0e\xcb\x02\x6b\xe8\xa4\x80\x23\xbd\xbd\x7b\x17\x49\xb5\x5d\x2a\x1c\xef\xbb\x54\x69\xb2\x72\xf9\xd0\xad\xce\x25\xc3\x15\x80\xee\x85\xf7\x88\xda\x9f\x46\xa4\x8a\xcf\x69\x94\x6f\x38\xee\x30\x75\x50\x44\xca\x96\xe9\x1e\x4d\xd4\xc4\x9c\x56\x17\x3f\x71\x19\x73\x99\xbc\x67\x41\xb7\xbc\x99\x41\x94\x80\x81\xf2\xa8\x04\x3c\xc2\xc2\x09\xaf\xd4\x36\x70\x97\x11\x21\xb5\x12\x7f\x78\x65\xb6\xd9\xeb\x3f\x81\xa1\xcf\x60\xbd\xfc\xe6\x7b\xd0\x92\x5a\x16\xff\x18\x2b\x9c\xaa\xec\x7a\xf9\x39\xaa\x6a\xbc\xbf\xda\xdb\x7c\xf5\x03\x89\xaa\xd5\xc0\x9c\x2c\xad\x0c\x6e\x48\x84\xc1\x88\x5c\x5d\x5e\x5e\x5e\x7a\x10\x48\x4d\x02\x38\x6f\x8c\x5a\x10\xc0\x50\x99\x02\x26\xd5\x7a\x97\xc8\x13\xee\xf0\x81\xda\x4a\x01\x0d\x67\xf6\x78\x55\x5d\xf5\xaa\xea\xea\xbd\x55\x45\x08\xe6\x1a\x22\x72\xaf\x62\x70\x52\x3b\xba\x73\x4c\x6f\x1b\xcb\x4f\x48\x11\x16\x99\x78\x82\x8f\xec\x8d\x2a\x3d\x56\x7c\x33\x22\x57\x1d\x35\xa5\xae\x5a\x7c\xa9\xc1\xda\xa3\xb9\x93\x70\xdb\xc2\x9f\xee\x07\xe0\x23\xa4\x5a\x50\x84\x12\x55\x4d\x67\xee\x4f\x34\x00\xee\x85\x78\x12\xc8\x9d\x1b\xbf\xc2\xeb\x52\xa9\x55\x44\x78\x22\x95\x01\xbf\xb4\x52\xad\x07\xb3\x58\x70\xc9\x31\xdf\xc2\x73\x6c\xe0\xba\x33\xea\x0c\xf1\x47\xc6\x0d\xc4\xb7\x99\xe1\x32\x79\x62\x4b\x88\x33\xc1\x65\x32\xf3\x07\x97\xc3\xd3\x37\x60\x99\xc3\x54\xdf\x59\x9c\xf9\x54\x9a\xee\x19\x4c\x6a\x9b\xd3\x41\x61\xc9\xe9\x9b\x36\x60\x9d\x4b\xb6\xe6\xdd\x8a\x15\xe4\x51\xdf\x05\x7b\xf4\x57\xfd\x29\x0d\x86\x3a\x67\x21\xf7\x0a\x67\xb2\x33\xbf\xa6\x22\x83\x8e\x28\xcf\x92\xa4\xba\xe8\x1b\x5e\x50\x61\xa1\x9a\x71\x92\x29\x97\x60\x36\x47\xb8\x2e\x34\x4d\xa9\x8c\xb7\x67\x06\x24\xcc\xac\x09\x85\x62\x54\x84\xaf\x5c\x86\x1b\xc3\xc7\x86\xaf\x6b\xa0\x03\x12\xac\x27\x7f\xad\x7f\x4d\x55\x0c\x93\x9e\xeb\x05\x24\xa8\x9a\xe9\x49\x26\xf9\x5b\x14\x86\x21\xb3\x3c\x6c\x7a\xd3\xd8\x2a\xb6\xaa\xef\x31\x90\x70\x8b\x26\x9f\x56\x7b\x91\xe9\x28\x0c\x2f\xc7\xfe\x5f\xb4\xcd\xe4\x95\xf8\x22\xcb\x7d\xf1\xdd\xc7\x24\xda\xa6\xaf\x72\xde\xd9\x94\xc7\x93\x1f\x7e\xfc\xc7\x6f\x9f\xa6\x2f\xf7\x0f\xb7\xd3\x97\xfb\xeb\xbb\xe9\x4f\xf5\x35\x8c\xfe\xca\x05\x4c\x42\x06\x06\x6d\xc8\xe8\x98\x19\x6c\xcc\x83\xc1\xfa\x0a\x14\xb6\xbd\x64\x05\x79\x7b\xc5\x0a\xf2\xfa\x8a\x42\x91\x2e\x85\x4c\x7e\xf8\x71\x7e\x37\xbd\x7b\xb9\x79\x9a\xbd\xdc\x3e\xce\x7e\x9f\x3e\xb6\x20\x81\x5c\xd7\x4d\x53\xa4\xa4\x26\xfe\x51\xcb\x3f\x7e\x35\x2a\x6d\xba\xc8\x82\x83\x88\x4b\xaa\x51\xff\x6b\x55\xa3\xe6\xa4\xdf\x34\xa7\xb8\x8c\x7c\xf8\x8d\x9d\xf6\x5c\x4a\xe9\xa0\x79\x9e\x3e\xde\xcd\xee\xaf\x9f\x67\x0f\xf7\x2f\x5f\x1e\x3e\xbf\xcc\xaf\x9f\xff\xde\xc6\x14\x91\x10\x53\x1d\x22\x98\xb4\x7c\xe2\x11\x08\x95\x74\x8e\xea\xd3\x45\xf7\xa8\x6e\x4c\x6d\xd6\xf0\x94\x26\x10\x11\x3f\xd1\x76\xdc\x88\x51\x49\x4d\xde\x5c\x3b\xcf\x84\x98\x2b\xc1\x59\x1e\x91\xd9\xe2\x5e\xe1\xdc\x80\x05\xb9\x35\x68\xad\x08\xb4\xfc\xdf\x02\xcb\x0c\xc7\xfc\xc6\xb5\x4c\x6f\xd8\xcc\x3c\x34\x7e\x90\x22\x7f\x54\xca\x7b\x8b\xcd\x2d\x42\x1a\x11\x34\xd9\x56\x7f\x35\x65\xdc\x81\xb5\x0e\x8c\xd7\xf6\xa0\xa6\x8a\xf6\xe4\xce\xf1\x35\x5b\x77\x8c\xd4\x8d\x94\xfb\xbd\xdf\xd5\xd0\x14\x57\xa8\xa2\xc9\xbb\xf0\xae\x9d\x96\x77\xf6\x69\x91\x25\x5c\x06\x2e\x38\x01\x83\x98\x9b\x1d\x7b\x31\xd5\x9d\xbd\x98\xea\xda\x8e\x80\x50\x93\x34\x50\x07\x41\xf9\x60\x29\xaf\xe5\x81\x60\x5b\x3a\xca\x8c\x02\x12\x0d\x15\x03\x89\xc7\x5b\x99\xc6\xb1\x4b\xc6\x93\x43\x52\x4b\xb0\x00\x8a\x99\x81\x20\xa1\x08\x76\xf2\xac\xb4\x12\x2a\xc9\x27\x0d\x03\xb9\x75\xd6\xa5\x14\x0c\x70\xe7\x02\xe4\x29\xa8\x0c\x27\x7f\x4b\x87\x63\x76\xfe\x70\xeb\xdd\xf9\x69\x7e\x7d\xf3\x0d\x21\x5b\x8b\xca\xaa\x6a\x8f\x37\x5c\x64\xa7\xd0\xf3\xcb\x6b\x47\xdc\x1f\x19\xcd\x5d\x7f\xb5\xfa\xc5\x56\xea\xaf\xb5\x20\xd1\xfa\x67\x97\xb2\x4f\x0a\xbd\xbe\x96\xe6\x6c\x31\x78\x50\x38\x1d\x11\x14\x3b\xf1\x98\x4c\x5e\xdb\x7b\x25\x1d\x9a\x16\x06\x3f\xf5\x9b\x05\x53\xb4\x21\x9b\x93\xea\xbd\xd9\x10\x97\xdb\xde\xa3\x56\xd6\x21\xd5\x98\xdf\x72\x13\x11\x99\x09\xd1\xce\x66\x3b\xd0\x07\x83\xa9\xc2\x02\x33\xd0\x50\x71\x31\xd2\xc2\xb6\xd9\x5c\xcc\xda\x1e\x48\xff\xfa\x77\x0b\x50\x95\x27\x86\x69\xfe\x2d\x85\x54\xc9\xf7\x25\xf9\xae\xca\x0d\xd3\xfb\x23\x09\x7d\x79\xe0\x89\xe8\xbe\x85\xa7\xd7\x24\x7f\x5f\x86\x7e\x76\x9a\x19\x83\x0b\x85\x3b\x2a\x69\x02\x66\x52\x60\xdd\x47\x43\x1b\xb7\x1f\x20\xa0\x9d\xd2\x70\x10\x4b\xdc\xc4\x5f\x8b\x9b\x96\xbb\x5c\xde\x9d\xcd\x7f\xf2\x14\xf5\xea\x00\x4a\xdb\x13\xd8\x5d\x7a\xfb\x5d\xa8\xa9\x75\xad\xb4\x4b\x7c\x93\x70\x4d\x4d\x28\xf8\x6b\xb8\x8f\xa2\x1e\x47\x68\x03\x12\xb8\xdb\xce\xc1\xb8\xb2\x4e\x13\x98\x5c\xd5\xae\xf9\x3f\x40\x77\x6b\x06\xfe\x0e\x50\x90\x62\x66\xc7\x5a\xc5\x35\x69\x67\x63\xcb\x67\x64\xf0\xff\x25\xc4\x5b\x1b\xbe\xe6\x02\x12\x88\x5b\x45\xb6\x51\x66\x2f\xdf\x9f\x83\x57\xc1\xb3\xca\x5e\x41\x00\x86\x45\xc9\xb5\xfe\xbb\x91\x80\xe0\x9f\x46\x37\xa9\x45\xb1\xdf\x28\x4d\x13\x2f\x34\x22\x9f\x78\x91\xec\xb8\x92\x35\xfe\x5b\xe9\xc6\xaf\xf7\x69\x64\x80\x96\x77\x80\x14\xaf\x52\x4f\x95\xe9\xb6\x0f\x48\xeb\x6f\x3d\x5c\x2c\x0d\xb5\x1d\x31\xac\x3b\x7b\x62\x58\x0f\xc8\xb1\x79\x57\x8a\xcd\x07\x91\xf5\xb0\xb8\xbd\x3d\x4d\xa5\xbc\x81\x18\x3a\x41\x85\xee\x30\x9f\x67\x87\x7b\xa3\x76\x8f\x53\x5a\xb0\xaa\x21\x85\x5f\xea\x46\xaa\x6e\x7b\xdb\x8e\x64\xdc\x5b\xf7\x7a\x7b\xa8\xe6\xaa\xfe\xc6\xe6\x4c\x3d\x7b\x4f\x07\xe1\xfd\xa6\x08\xfe\xcd\xad\x4d\xb4\xbe\x1a\xff\x7c\x62\x2f\xd1\x3e\xeb\xc3\xfb\x88\xbd\x1e\x58\x37\xf6\xae\x96\xbe\xf0\x84\xed\x19\xf5\x67\x97\x5b\x30\xe5\x8b\xcf\xc2\x14\xbb\xda\x86\xa5\xb2\x85\xe4\x7a\x4e\x1d\xce\x68\x83\xb1\x51\x3c\xc6\xbf\xf5\xd1\xa0\x4c\xfe\x60\x6e\xaa\xf7\xe4\x83\x5a\x38\x05\xc8\x4b\xc5\xad\xc2\xa3\xe5\xef\xd0\xe3\x49\xea\x18\x4a\xf0\x87\xa1\xe9\x4d\xeb\xc7\x61\x69\xe6\xf8\xc3\xc4\xb6\x32\x7b\xb0\x33\x77\x1f\xd8\x07\xfa\x8d\xed\x1e\xf0\x80\x4b\x9c\xc1\x9f\x76\x64\xd8\x01\xe1\xcd\x0a\x74\x98\x94\x66\x85\x1a\x3c\xbe\x59\xae\x0e\x0c\x8a\xb2\x9c\xb5\x7b\xe2\xe6\x0f\x18\xc2\xf5\xd5\x2b\x20\xdd\xbc\xd5\x7e\x9a\xdd\x16\x64\xe9\xdd\x5a\xe4\x9a\x5d\xaa\xa6\xaf\xf8\xb5\xc4\x63\xf9\xba\x25\x22\xfe\xa5\x83\x7f\xa3\x18\xcf\xe4\x42\x3d\x48\x9f\x18\x37\x29\xb3\xbc\xc1\x4d\xf9\xe8\x6f\x33\x5e\x64\xa3\x2f\x7c\x01\x2c\x67\x02\xee\x54\x5c\xfd\x28\x63\xbe\xf9\x4d\x91\xff\x3a\xd5\x4b\x48\xc1\x50\x31\xfa\x4f\x00\x00\x00\xff\xff\x31\x1f\x86\x66\x5f\x29\x00\x00") +var _deployKubernetes119AlphaDirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\x6e\xef\xe1\x0e\xa8\xac\xf8\x8a\x02\x07\x01\x7e\xc8\x26\xbe\xad\xd1\x4d\x62\x24\x7b\xf7\x1a\x30\xd4\x58\x66\x4d\x91\x3c\x72\xe4\x8d\xae\xe8\xff\x5e\x90\xfa\xb0\xbe\xac\xd8\xbe\x64\x83\x9e\xf7\x61\x1d\x89\xe4\xfc\xe6\xeb\x37\x33\x96\xfe\x4a\x3e\x81\x04\x43\x11\x62\xf2\x95\xe3\x86\x7c\x48\xe9\x16\xc8\x36\xb3\xa8\x52\xfe\x3b\x7c\xf8\x1b\x89\x15\x91\x0a\x09\xc4\x1c\xff\x32\x99\x50\xcd\x7f\x05\x63\xb9\x92\x11\xd9\xcd\x26\x5b\x2e\xe3\x88\x3c\x80\xd9\x71\x06\x97\x8c\xa9\x4c\xe2\x24\x05\xa4\x31\x45\x1a\x4d\x08\x11\xf4\x09\x84\x75\xdf\x08\xd1\x29\xa4\x01\xb3\x7c\xca\x25\x82\x98\x32\x95\x86\x31\x68\xa1\xf2\x14\x24\x46\x24\xe6\x06\x18\x06\xda\xa8\x38\x63\xc8\x95\x9c\x10\x22\x69\x0a\x51\xbd\x31\x60\x4a\xa2\x51\x42\x80\x29\xef\x59\x4d\x19\x44\x24\x86\x35\xcd\x04\x4e\x82\x20\x68\x21\x34\x4f\x94\x4d\x69\x86\x1b\x65\xf8\xef\xd4\x9d\x39\xdd\xfe\x64\xa7\x5c\x85\x35\xf6\x7b\x25\xe0\x0d\x11\xc3\x33\x82\x91\x54\xb8\x45\x3b\xee\x50\x81\x09\xd8\x3a\x19\xc6\x6f\x32\x01\x36\x9a\x04\x84\x6a\xfe\xc9\xa8\x4c\x7b\x1c\x01\xf9\xf0\x61\x42\x88\x01\xab\x32\xc3\xa0\xbc\x06\x32\xd6\x8a\x4b\xb4\x13\x42\x76\x60\x9e\xca\xcb\x09\xa0\xff\xff\x2b\x45\xb6\xf1\xdf\x04\xb7\xc5\xa5\x18\x04\x20\xf8\xaf\x99\x8e\x69\xf9\x95\x19\x70\x5f\x7b\x32\x99\x52\x26\xe6\xb2\x69\xb5\x3e\x08\x01\xd4\xc2\x5b\x21\xb0\xa8\x0c\x4d\xe0\xa0\x70\x66\x79\xb9\x84\x51\x4d\x19\x47\x7e\x08\x4a\x0d\x60\x8f\xa9\x14\xda\x81\xa2\xeb\xfb\x25\xd4\xa3\x5c\xa1\x55\x3c\x20\xb8\xb7\x95\x6a\x6d\xfb\x9b\x0d\x68\xc1\x19\xb5\x30\xe4\xc9\x33\x02\xfa\x4a\x64\x16\xc1\xbc\x47\x5c\x9b\x4c\x4a\x30\x27\x45\xb1\x76\x9a\x59\x04\x89\x3b\x25\xb2\xf4\x3c\x07\x9e\xe4\xab\x8e\x40\x26\x28\x4f\x8f\x97\x5a\xc6\xca\xc9\xb1\x5a\x05\xaa\xa0\xf6\x60\xc2\x74\xc4\x1d\xc7\x02\x3b\xe8\x51\xc0\x89\xd1\xde\xd7\x45\x52\x6d\x37\x0a\xa7\x2f\x29\x55\xba\xac\x5c\x3e\xa6\xd5\x6b\xc9\x70\x05\xa0\xaf\xf0\x0b\xa2\x5e\xa6\x11\xa9\xe2\xd7\x74\xca\x1f\x38\xee\x38\x73\x50\x44\xca\x36\xe9\x0b\x96\x68\x88\x39\xaf\x2e\x7e\xe4\x32\xe6\x32\x79\xcb\x82\x6e\x79\x9b\x41\x94\x80\x91\xf2\xa8\x04\xdc\xc3\xda\x09\xaf\xcc\x36\xa2\xcb\x84\x90\x46\x89\x3f\xbe\x32\xdb\xec\xe9\xdf\xc0\xd0\x33\xd8\x60\x7f\xf3\x2d\xda\x92\x06\x8b\xbf\x8f\x17\xce\x35\x76\xb3\xfc\x9c\x54\x35\xde\xde\xec\xdd\x7e\xf5\x1d\x1b\x55\xab\x81\x39\x59\x5a\x19\xac\x9b\x08\x83\x11\x99\x5d\x5c\x5c\x5c\x78\x10\x48\x4d\x02\xb8\x6a\x5d\xb5\x20\x80\xa1\x32\x05\x4c\xaa\xf5\x21\x91\x67\xe8\xf0\x8e\xd6\x4a\x01\x0d\x67\xf6\x74\x53\xcd\x06\x4d\x35\x7b\x6b\x53\x11\x82\xb9\x86\x88\xdc\xaa\x18\x9c\xd4\x9e\xed\x5c\xa7\xb7\xcf\xe5\x07\xa4\x08\xeb\x4c\x3c\xc0\x7b\xce\x46\x95\x1d\xab\x7e\x33\x22\xb3\x9e\x99\x52\x57\x2d\x3e\x37\x60\xbd\x60\xb9\xb3\x70\xdb\x22\x9e\x6e\x47\xe0\x23\xa4\x5a\x50\x84\x12\x55\xc3\x66\x1e\x92\x94\x0a\x3d\xf9\xd4\x28\x07\x81\x58\x66\xa8\x73\x93\x3b\x9a\x72\x09\xc6\x96\xab\x45\x4b\xc3\x17\x75\x3c\x4b\xcb\x83\x1b\xbf\xc2\xd3\x46\xa9\x6d\x44\x78\x22\x95\x01\xbf\xb4\xf2\x8d\x07\xb3\x5e\x73\xc9\x31\xdf\xc3\x73\xed\xc4\x65\xef\xaa\xf3\xe4\x6f\x19\x37\x10\x5f\x67\x86\xcb\xe4\x81\x6d\x20\xce\x04\x97\xc9\xd2\x1f\x5c\x5e\x5e\x3c\x03\xcb\x1c\xa6\xe6\xce\xe2\xcc\x87\xd2\xf7\x5f\xc0\xa4\xb6\x7d\x3b\x28\x42\x61\xf1\xac\x0d\x58\xdb\xb6\x74\xb5\x62\x0b\x79\x34\xa4\xe0\x80\xfd\xaa\x8f\xd2\x60\xa8\x8b\x36\x72\xab\x70\x29\x7b\xf7\x77\x54\x64\xd0\x13\xe5\xdb\x2c\xa9\x3e\x0c\x5d\x5e\x53\x61\xa1\xba\xb3\x77\x74\x75\x84\x1b\x63\xd3\x94\xca\x78\x7f\x66\x40\xc2\xcc\x9a\x50\x28\x46\x45\xf8\xc4\x65\x58\x3b\x3e\x36\x7c\xd7\x00\x1d\x90\x60\x37\xff\x7b\xf3\xcf\x54\xc5\x30\x1f\x50\x2f\x20\x41\x35\x8d\xcf\x33\xc9\x9f\xa3\x30\x0c\x99\xe5\x61\x3b\x9a\xa6\x56\xb1\x6d\x73\x8f\x81\x84\x5b\x34\xf9\xa2\xda\x8b\x4c\x47\x61\x78\x31\xf5\xff\xa2\x7d\x29\x28\xd7\x3b\x9f\xf1\x78\xfe\xdd\xf7\xff\xfa\xe5\xe3\xe2\xf1\xf6\xee\x7a\xf1\x78\x7b\x79\xb3\xf8\xa1\xb9\x86\xd1\x9f\xb9\x80\x79\xc8\xc0\xa0\x0d\x19\x9d\x32\x83\xad\xfb\x60\xb0\xb9\x02\x85\xed\x2e\xd9\x42\xde\x5d\xb1\x85\xbc\xb9\xa2\x30\x94\xe3\x98\xf9\x77\xdf\xaf\x6e\x16\x37\x8f\x57\x0f\xcb\xc7\xeb\xfb\xe5\xaf\x8b\xfb\x3e\xa4\x92\xdd\x3f\xfb\xa9\x6b\x1e\xed\x69\xdb\x7d\x40\xee\x9a\xae\x29\x38\xad\xad\xdf\xa4\x13\x1f\x3f\x1b\x95\xb6\x43\x64\xcd\x41\xc4\x65\xaf\xd2\xfc\x74\xca\x59\xfb\xa6\xdf\xb4\xa2\xb8\x89\x7c\xfa\x4d\x9d\x75\x1d\x27\xf5\xd0\x7c\x59\xdc\xdf\x2c\x6f\x2f\xbf\x2c\xef\x6e\x1f\x3f\xdf\x7d\x7a\x5c\x5d\x7e\xf9\x67\x17\x53\x44\x42\x4c\x75\x88\x60\xd2\xf2\x27\x93\x40\xa8\xa4\x77\xd4\x90\xad\xfa\x47\xf5\x73\xaa\x5e\xc3\x53\x9a\x40\x44\xfc\x8d\x6e\xe0\x46\x8c\x4a\x6a\xf2\xf6\xda\x55\x26\xc4\x4a\x09\xce\xf2\x88\x2c\xd7\xb7\x0a\x57\x06\x2c\xc8\xbd\xc3\x1b\x55\xa4\x13\xff\x75\xc1\xad\x74\xa8\xd3\x6b\xd5\xa9\xbf\xcd\x93\xf6\xb5\xbc\xf8\x58\x60\x99\xe1\x98\x5f\xb9\xc9\xed\x19\xdb\xfc\x45\xe3\x3b\x29\xf2\x7b\xa5\x7c\x4c\xda\xdc\x22\xa4\x11\x41\x93\xed\xbd\xd0\x30\xe9\x0d\x58\xeb\x54\xf2\x3e\x1b\xb5\x77\x31\x25\xdd\xb8\xb6\xb1\xa5\x41\xea\xae\x94\xfb\x7d\x74\xf7\xe0\x57\x39\xe9\x13\xe5\xd0\x4e\xcb\x7b\xfb\xb4\xc8\x12\x2e\x03\x97\xe2\x80\x41\xcc\xcd\x81\xbd\x98\xea\xde\x5e\x4c\x75\x63\x47\x40\xa8\x49\x5a\xa8\x83\xa0\xfc\x7d\x2b\x6f\xb0\x49\xb0\x2f\x40\x25\x2f\x81\x44\x43\xc5\x08\x7d\xf9\x58\xa1\x71\xec\x28\x7d\x7e\x0c\x41\x05\x6b\xa0\x98\x19\x08\x12\x8a\x60\xe7\x5f\x94\x56\x42\x25\xf9\xbc\xe5\x20\xb7\xce\x3a\x9f\x63\x80\x07\x17\x20\x4f\x41\x65\x38\xff\x47\xda\xba\x5c\x06\x4b\x0d\xca\x93\xc3\x6c\x9c\x1c\x56\x77\xd7\x3e\x6f\x1e\x56\x97\x57\x7f\x80\x1b\x1a\xe9\x5f\xf5\x17\xd3\xba\x6b\x3a\x28\xf4\xf5\xe5\x75\x53\xfb\xb7\x8c\xe6\x6e\x12\xdc\xfe\x64\x2b\x0f\x35\x86\xa5\x68\xf7\xa3\xab\x0d\x67\xe5\xf8\xd0\xf0\x75\x52\xb2\xcf\xde\x3c\xd9\x8f\xca\xdb\x13\xb2\xef\x20\x1e\x93\xc9\x4b\x7b\xab\xa4\x43\xd3\xc1\xe0\x6f\xfd\x62\xc1\x14\x63\x57\x7d\x52\x73\x16\x1d\xeb\x5d\xf7\x7a\x34\xba\x10\x48\x35\xe6\xd7\xdc\x44\x44\x66\x42\x74\xc9\xf7\x00\xfa\x60\x94\x93\x2c\x30\x03\x2d\x13\x17\x57\x3a\xd8\xea\xcd\xc5\x5d\x3b\x00\xe9\x3f\xff\xed\x00\xaa\x08\x69\x7c\xac\xb9\xa6\x90\x2a\xf9\xb6\x43\x8d\x2b\xca\xe3\xe3\xcc\x89\x03\x4c\x79\xe0\x99\xe8\xde\x75\x2e\x69\x40\xff\xb6\x13\xc9\xab\xb7\xd5\x31\xb8\x5c\xba\xa1\x92\x26\x60\xe6\x05\xd6\x97\xda\xee\x96\xf6\x23\x0d\x77\xaf\x88\x1d\xd5\x35\xd7\x09\xdc\xe9\xc5\xcb\x5d\x8e\xfe\x97\xab\x1f\x7c\x4b\x3e\x3b\xa2\x85\x1f\x60\x86\x7e\x3b\xff\x4d\x5a\x75\x8b\x14\x7d\xc7\x34\x0f\x77\xd4\x84\x82\x3f\x85\x47\xb4\xec\xa7\x35\xf8\x4e\xdb\x15\x18\xd7\x80\xd0\x04\xe6\xb3\xb6\x9a\xff\xef\xed\x7f\x23\x00\xbe\x01\x14\xa4\x98\xd9\xa9\x56\x71\x43\xda\xab\x4d\x0f\xaf\x38\xd1\xfc\xa9\x06\x11\x6d\xf8\x8e\x0b\x48\x20\xee\xf4\x02\xad\x6e\xe0\xe2\xed\x67\x92\x2a\x45\xb7\xd9\x13\x08\xc0\xb0\xe8\x0c\xac\xff\xdb\x48\x40\xf0\x0f\x09\xda\x1d\x50\xb1\xdf\x28\x4d\x13\x2f\x34\x22\x1f\x79\x41\xa9\x5c\xc9\xc6\x3c\x50\xdb\xc5\xad\xf7\x64\x35\x32\xa6\xf4\x80\x14\x4f\xb8\xcf\x95\xe9\xb6\x8f\x48\x1b\x1e\xc5\x5c\x46\x8e\x8d\x61\x31\xec\x7a\x7b\x62\xd8\x8d\xc8\xb1\x79\x5f\x8a\xcd\x47\x91\x0d\x34\x9b\x2f\xce\x78\x95\xf1\x46\x32\xf1\x0c\x13\xba\xc3\x3c\x9b\x8f\xcf\x8a\xdd\x99\xaf\xf4\x60\x55\xa9\x8a\xb8\xd4\xad\x82\xd0\x8d\xb6\x03\x94\x3f\x58\x5d\x07\x67\xca\xf6\xaa\xe1\x29\xee\x95\x7e\x09\x19\x18\x97\x7c\xdc\x14\x14\x52\x6b\x6d\xa2\xdd\x6c\xfa\xe3\x99\x83\x53\xf7\xac\x77\x1f\x77\x5e\x8c\xc0\xa6\xb3\x7b\xbb\x5b\x91\xb0\x3f\xa3\xf9\x8b\xf0\x1e\x4c\xf9\x3c\xba\x70\xc5\xa1\xe9\x66\xa3\x6c\x21\xb9\xc9\xa9\xe3\x8c\x36\x9a\x1b\xc5\xd3\x95\x6b\x9f\x0d\xca\xe4\x77\xe6\xaa\x7a\x7d\x61\xd4\x0a\xe7\x00\x79\xac\x3a\xb8\xf0\x64\xf9\x07\xec\x78\x96\x39\xc6\x08\xfe\x38\x34\x83\xb4\x7e\x1a\x96\x36\xc7\x1f\x27\xb6\xc3\xec\xc1\x41\xee\x3e\x72\x5c\xf5\x1b\xbb\xa3\xea\x11\x4a\xbc\x42\x3c\x1d\x60\xd8\x11\xe1\xed\x0a\x74\x9c\x94\x76\x85\x1a\x3d\xbe\x5d\xae\x8e\x4c\x8a\xb2\x9c\x75\x47\xf7\xf6\x7b\x25\xe1\x6e\xf6\x04\x48\xeb\x97\x0d\x1e\x96\xd7\x45\xcb\xf5\x66\x93\x7c\xc3\x2f\xd5\x68\x59\xbc\xc4\x72\x5f\x3e\xc4\x8a\x88\x7f\x94\xe3\x1f\xf4\xc6\x4b\xb9\x56\x77\xd2\x13\x63\x4d\x99\xa5\x06\x57\xe5\x4f\xa1\xf5\xf5\x82\x8d\x3e\xf3\x35\xb0\x9c\x09\xb8\x51\x71\xf5\xae\xcc\xaa\x7e\xd5\xcb\xff\xb9\xd0\x1b\x48\xc1\x50\x31\xf9\x5f\x00\x00\x00\xff\xff\x10\xe9\xf0\xfc\xf6\x2a\x00\x00") func deployKubernetes119AlphaDirectPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -179,12 +179,12 @@ func deployKubernetes119AlphaDirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml", size: 10591, mode: os.FileMode(420), modTime: time.Unix(1599471431, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml", size: 10998, mode: os.FileMode(420), modTime: time.Unix(1599822744, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119AlphaLvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xd3\x7d\xd8\x05\x2a\x2b\xd9\xa2\xc0\x42\x80\x1f\x32\x89\x77\x6a\x74\x92\x18\x49\x76\x5f\x03\x9a\x3a\x96\x59\x53\x24\x97\xa4\x34\xd1\x14\xfd\xef\x05\x29\xc9\xa6\x2e\x96\x2f\xe3\x24\xdd\xcc\xc3\xd8\xbc\x9d\x8f\xe7\xf2\x9d\x73\x24\xff\x15\x7d\x06\x0e\x0a\x1b\x88\xd1\x57\x6a\x56\xe8\x22\xc5\x6b\x40\xeb\x4c\x1b\x91\xd2\x6f\x70\xf1\x37\x14\x0b\xc4\x85\x41\x10\x53\xf3\x97\xd1\x08\x4b\xfa\x3b\x28\x4d\x05\x8f\x50\x7e\x35\x5a\x53\x1e\x47\xe8\x09\x54\x4e\x09\x5c\x13\x22\x32\x6e\x46\x29\x18\x1c\x63\x83\xa3\x11\x42\x0c\x2f\x80\x69\xfb\x09\x21\x99\x42\x1a\x10\x4d\xc7\x94\x1b\x60\x63\x22\xd2\x30\x06\xc9\x44\x91\x02\x37\x11\x62\x79\x1a\x48\x25\xe2\x8c\x18\x2a\xf8\x08\x21\x8e\x53\x88\x36\xbb\x02\x22\xb8\x51\x82\x31\x50\xd5\x9c\x96\x98\x40\x84\x62\x58\xe2\x8c\x99\x51\x10\x04\x0d\x78\x6a\x81\xc9\x18\x67\x66\x25\x14\xfd\x86\xed\x99\xe3\xf5\x2f\x7a\x4c\x45\xb8\x01\xfe\x28\x18\xbc\x15\x5c\x78\x35\xa0\x38\x66\x76\x51\x4e\x2d\x24\x50\x01\x59\x26\xfd\xe0\x55\xc6\x40\x47\xa3\x00\x61\x49\x3f\x2b\x91\x49\x07\x22\x40\x17\x17\x23\x84\x14\x68\x91\x29\x02\xd5\x18\xf0\x58\x0a\xca\x8d\x1e\x21\x94\x83\x5a\x54\xc3\x09\x18\xf7\xff\x57\x6c\xc8\xca\x7d\x62\x54\x97\x43\x31\x30\x30\xe0\x3e\x66\x32\xc6\xd5\x47\xa2\xc0\x7e\xec\xc8\x24\x42\xa8\x98\x72\x5f\x65\x5d\x10\x0c\xb0\x86\xb7\x42\xa0\x8d\x50\x38\x81\x9d\xc2\x89\xa6\xd5\x12\x82\x25\x26\xd4\xd0\x5d\x50\x36\x00\xb6\x98\x2a\xa1\x2d\x28\x72\x33\x5f\x41\x3d\xc8\x14\x52\xc4\x3d\x82\x3b\x5b\xb1\x94\xba\xbb\x59\x81\x64\x94\x60\x0d\x7d\x96\x3c\xc1\x9b\x6f\x58\xa6\x0d\xa8\x77\x77\x6a\x95\x71\x0e\xea\x28\x17\x96\xf6\x5a\xda\x00\x37\xb9\x60\x59\x7a\x9a\xf5\x8e\x32\x54\x4b\x20\x61\x98\xa6\x87\x4b\xad\x1c\xe5\x68\x47\xad\xbd\x94\x61\xbd\x33\x5a\x5a\xe2\x0e\xa3\x80\x1c\x3a\xf1\x7f\xa4\xab\x77\xef\xc2\xb1\xd4\x2b\x61\xc6\xfb\x2e\x55\x99\xac\x5a\x3e\x74\xab\x73\xc9\xb0\xd4\xdf\xbd\xf0\x1e\x51\xfb\x39\x84\x8b\xf8\x9c\x46\xf9\x8e\xe3\x0e\x53\x07\x36\x06\x93\x55\xba\x47\x13\x9e\x98\xd3\x32\xe2\x27\xca\x63\xca\x93\x37\xcb\xe3\x9a\x36\xe9\x43\x30\x18\x48\x8c\x82\xc1\x23\x2c\xad\xe4\x5a\x67\x03\x17\x19\x21\xe4\x65\xf6\xc3\x73\xb2\xce\x16\xff\x06\x62\x1c\x7d\xf5\xd6\x34\xef\x51\x8d\x78\xfc\xfd\x01\x26\x38\x55\xd3\x7e\xd6\x39\x2a\x5f\xbc\xbd\xce\xdb\x05\xea\x47\x55\xa6\x5a\x02\xb1\x82\xa4\x50\x66\x53\x38\x28\x13\xa1\xab\xcb\xcb\xcb\x4b\x87\xc0\x60\x95\x80\x99\x37\x46\x35\x30\x20\x46\xa8\x12\x23\x96\x72\x97\xc8\x63\x2f\xf0\x51\x7a\x4a\xc1\x28\x4a\xf4\xf1\x4a\xba\xea\x55\xd2\xd5\x9b\x2a\x09\x21\x53\x48\x88\xd0\xbd\x88\xc1\x8a\xec\x68\xcd\x16\x75\xdb\xe0\x7d\x32\xd8\xc0\x32\x63\x4f\xf0\x61\x0d\x50\xad\xc1\xba\xae\x8c\xd0\x55\x47\x41\xa9\x4d\x0c\x5f\x3c\x4c\x7b\x74\x76\x3c\x68\x5d\xfa\xd0\xfd\x00\x76\x03\xa9\x64\xd8\x40\x05\xc9\xd3\x96\xfd\x63\x0d\x74\x7b\xf1\x1d\x8f\x70\xe7\xae\xaf\xb0\x58\x09\xb1\x8e\x10\x4d\xb8\x50\xe0\x96\xd6\x4a\x75\x48\x96\x4b\xca\xa9\x29\xb6\xd8\x6c\xca\xbf\xee\x8c\x5a\x13\xfc\x91\x51\x05\xf1\x6d\xa6\x28\x4f\x9e\xc8\x0a\xe2\x8c\x51\x9e\xcc\xdc\xc1\xd5\xf0\xf4\x15\x48\x66\x31\xf9\x3b\xcb\x33\x9f\x2a\xa3\x3d\x83\x4a\x75\x73\x3a\x28\x6d\x38\x7d\x95\x0a\xb4\xf5\xc4\xd6\xbc\x5d\xb1\x86\x22\xea\xbb\x60\x8f\xf2\xea\x3f\x21\x41\x61\xeb\x26\xe8\x5e\x98\x19\xef\xcc\xe7\x98\x65\xd0\x11\xe5\x4a\x21\x2e\x2e\xfa\x86\x97\x98\x69\xa8\x67\xac\x64\x4c\x39\xa8\xcd\x11\xb6\xcf\x4c\x53\xcc\xe3\xed\x99\x01\x0a\x33\xad\x42\x26\x08\x66\xe1\x82\xf2\x70\x63\xf5\x58\xd1\xdc\x03\x1d\xa0\x20\x9f\xfc\xdd\xff\x9a\x8a\x18\x26\x3d\xd7\x0b\x50\x50\xb7\xcb\x93\x8c\xd3\xd7\x28\x0c\x43\xa2\x69\xd8\x74\xa5\xb1\x16\x64\xed\xef\x51\x90\x50\x6d\x54\x31\xad\xf7\x1a\x22\xa3\x30\xbc\x1c\xbb\x7f\xd1\x96\xb7\x6b\xf1\x25\xb3\x7d\x71\x2d\xc6\x24\xda\x52\x56\x35\x6f\x6d\x4a\xe3\xc9\x0f\x3f\xfe\xeb\xb7\x4f\xd3\x97\xfb\x87\xdb\xe9\xcb\xfd\xf5\xdd\xf4\x27\x7f\x0d\xc1\xbf\x52\x06\x93\x90\x80\x32\x3a\x24\x78\x4c\x94\x69\xcc\x83\x32\xfe\x0a\xc3\x74\x7b\xc9\x1a\x8a\xf6\x8a\x35\x14\xfe\x8a\x52\x91\x96\x3c\x26\x3f\xfc\x38\xbf\x9b\xde\xbd\xdc\x3c\xcd\x5e\x6e\x1f\x67\xbf\x4f\x1f\x5b\x90\x80\xe7\xbe\x69\x4a\x32\x6a\xe2\x1f\xb5\xfc\xe3\x57\x25\xd2\xa6\x8b\x2c\x29\xb0\xb8\xaa\x2a\xfc\xbf\x56\xfa\x69\x4e\xba\x4d\x73\x6c\x56\x91\x0b\xbf\xb1\xd5\x9e\xe5\x93\x0e\x9a\xe7\xe9\xe3\xdd\xec\xfe\xfa\x79\xf6\x70\xff\xf2\xe5\xe1\xf3\xcb\xfc\xfa\xf9\x9f\x6d\x4c\x11\x0a\x4d\x2a\x43\x03\x2a\xad\x9e\x69\x04\x4c\x24\x9d\xa3\xfa\x74\xd1\x3d\xaa\x1b\x53\x9b\x35\x34\xc5\x09\x44\xc8\x4d\xb4\x1d\x37\x22\x98\x63\x55\x34\xd7\xce\x33\xc6\xe6\x82\x51\x52\x44\x68\xb6\xbc\x17\x66\xae\x40\x03\xdf\x1a\xd4\xa3\xff\x96\xff\x6b\x20\x99\xa2\xa6\xb8\xb1\x7d\xd1\xab\x69\x32\x0f\x8e\x1f\x38\x2b\x1e\x85\x70\xde\xa2\x0b\x6d\x20\x8d\x90\x51\xd9\x56\x7f\x9e\x32\xee\x40\x6b\x0b\xc6\x69\x7b\x50\x53\x65\x0f\x72\x67\x4b\x33\xed\x3b\x46\x6a\x47\xaa\xfd\xce\xef\x3c\x34\xe5\x15\xea\x68\x72\x2e\xbc\x6b\xa7\xa6\x9d\x7d\x92\x65\x09\xe5\x81\x0d\x4e\x30\x41\x4c\xd5\x8e\xbd\x26\x95\x9d\xbd\x26\x95\xde\x8e\x00\x61\x95\x34\x50\x07\x41\xf5\xe8\xa8\xf0\x78\x20\xd8\xe6\x8d\x8a\x51\x80\x1b\x85\xd9\x00\xf1\x38\x2b\xe3\x38\xb6\x64\x3c\x39\x84\x5a\x82\x25\x60\x93\x29\x08\x12\x6c\x40\x4f\x9e\x85\x14\x4c\x24\xc5\xa4\x61\x20\xbb\x4e\x5b\x4a\x31\x81\xd9\xb9\xc0\xd0\x14\x44\x66\x26\xff\x48\x87\x63\x76\xfe\x70\xeb\xdc\xf9\x69\x7e\x7d\xf3\x1d\x21\xeb\x45\x65\x9d\xb2\xc7\x9b\x2a\x64\xa7\xd0\xf3\xcb\x6b\x47\xdc\x1f\x19\x2e\x6c\x1f\xb5\xfe\x45\xd7\xea\xf7\xba\x8d\x28\xff\xd9\x52\xf6\x49\xa1\xd7\xd7\xbd\x9c\x2d\x06\x0f\x0a\xa7\x23\x82\x62\x27\x1e\x95\xf1\x6b\x7d\x2f\xb8\x45\xd3\xc2\xe0\xa6\x7e\xd3\xa0\xca\xa6\x63\x73\x92\xdf\x86\x0d\x15\x72\xdb\x7b\x78\x69\x1d\x52\x69\x8a\x5b\xaa\x22\xc4\x33\xc6\xda\x6c\xb6\x03\x7d\x30\x48\x15\x1a\x88\x82\x86\x8a\xcb\x91\x16\xb6\xcd\xe6\x72\x56\xf7\x40\xfa\xcf\x7f\x5b\x80\x6a\x9e\x18\xae\xee\x6f\x31\xa4\x82\xbf\x61\x6d\x6f\x53\xdc\x70\x55\x7f\x64\x1d\x5f\x1d\x78\x0a\xb4\xef\x29\xcf\x3d\xb1\xef\x58\x98\x9f\xbd\xba\x8c\xc1\x46\xc0\x1d\xe6\x38\x01\x35\x61\x79\xba\xaf\xf4\x6c\xdc\x7b\xa0\xe8\xec\xa4\x83\x83\x2a\xc3\x4d\xcc\xb5\xea\xd1\x6a\x97\xe5\xda\xd9\xfc\x27\x57\x96\x5e\x1d\x50\xc6\xf6\x04\x73\xb7\xa4\x7d\x97\x72\x54\xdb\xae\xd9\x92\xdd\x24\xcc\xb1\x0a\x19\x5d\x84\xfb\xca\xd2\xe3\x8a\xd8\x00\x05\xf6\xb6\x73\x50\x36\x95\xe3\x04\x26\x57\xde\x35\xff\x04\x25\xae\x67\xe0\x77\x80\x62\xb0\xc9\xf4\x58\x8a\xd8\x93\x76\xb6\x0a\xf9\x8c\x55\xfb\xff\x49\xb1\x2d\x15\xcd\x29\x83\x04\xe2\x56\x62\x6d\xa4\xd6\xcb\xb7\xaf\xbb\xeb\xe0\x59\x67\x0b\x60\x60\xc2\x32\xcd\x6a\xf7\x5d\x71\x30\xe0\x9e\x34\x37\xcb\x89\x72\xbf\x12\x12\x27\x4e\x68\x84\x3e\xd1\x98\x2a\x70\x6c\xec\xd5\xbc\xb5\x6e\xdc\x7a\x47\x23\x03\xa5\x78\x07\x48\xf9\x82\xf4\x54\x99\x76\xfb\x80\xb4\xfe\x76\xc3\xc6\xd2\x50\xab\x11\x43\xde\xd9\x13\x43\x3e\x20\x47\x17\x5d\x29\xba\x18\x44\xd6\x53\xb9\xed\xed\x63\x6a\xe5\x0d\xc4\xd0\x09\x2a\xb4\x87\x39\x9e\x1d\xee\x87\xda\x7d\x4d\x65\xc1\x3a\x87\x94\x7e\x29\x1b\x54\xdd\xf6\xb6\x1d\x64\xdc\x9b\xf7\x7a\xfb\xa6\xe6\xaa\xfe\x66\xe6\x4c\x7d\x7a\x4f\xd7\xe0\xfc\xa6\x0c\xfe\xcd\xad\x55\x94\x5f\x8d\x7f\x3e\xb1\x7f\x68\x9f\xf5\xe1\xbd\xc3\x5e\x0f\xf4\x8d\xbd\xab\x8d\x2f\x3d\x61\x7b\x86\xff\xbc\x72\x0b\xa6\x7a\xa3\x59\x9a\x62\x57\xab\xb0\x12\xba\x94\xec\x73\xea\x30\xa3\x0d\xc6\x46\xf9\xc4\xfe\xd6\x45\x83\x50\xc5\x83\xba\xa9\x5f\x80\x0f\x6a\xe1\x14\x20\x2f\x75\x6d\x15\x1e\x2d\x7f\x87\x1e\x4f\x52\xc7\x10\xc1\x1f\x86\xa6\x97\xd6\x8f\xc3\xd2\xe4\xf8\xc3\xc4\xb6\x98\x3d\xd8\xc9\xdd\x07\xf6\x7e\x6e\x63\xbb\xef\x3b\xe0\x12\x67\xf0\xa7\x1d\x0c\x3b\x20\xbc\x99\x81\x0e\x93\xd2\xcc\x50\x83\xc7\x37\xd3\xd5\x81\x41\x51\xa5\xb3\x76\x1f\xdc\xfc\x65\x42\x98\x5f\x2d\xc0\xe0\xcd\x1b\xeb\xa7\xd9\x6d\x59\x2c\xbd\x4d\x5b\xec\x19\xa5\x6e\xf7\xca\xdf\x40\x3c\x56\xef\x57\x22\xe4\xde\x32\xb8\xd7\x86\xf1\x8c\x2f\xc5\x03\x77\xac\xb8\xe1\xcb\x0a\xfe\x4d\xf5\xac\x6f\x33\x5e\x52\xd1\x17\xba\x04\x52\x10\x06\x77\x22\xae\x7f\x6a\x31\xdf\xfc\x52\xc8\x7d\x9d\xca\x15\xa4\xa0\x30\x1b\xfd\x2f\x00\x00\xff\xff\xb5\x27\xe8\xb6\x2c\x29\x00\x00") +var _deployKubernetes119AlphaLvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\x6e\xef\xe1\x0e\xa8\xac\xf8\x8a\x02\x07\x01\x7e\xc8\x26\xbe\xad\xd1\x4d\x62\x24\xb9\x7b\x0d\x18\x6a\x2c\xb3\xa6\x48\x1d\x39\xd2\x46\x57\xf4\x7f\x2f\x48\x7d\x58\x5f\x56\x6c\x5f\x9c\xa0\xe7\x7d\x58\x47\x22\x39\xbf\xf9\xfa\xcd\x8c\xa5\xbf\x92\x2f\x20\x41\x53\x84\x90\x7c\xe3\xb8\x21\x9f\x62\xba\x05\xb2\x4d\x0d\xaa\x98\xff\x0e\x9f\xfe\x46\x42\x45\xa4\x42\x02\x21\xc7\xbf\x4c\x26\x34\xe1\xbf\x82\x36\x5c\xc9\x80\x64\xb3\xc9\x96\xcb\x30\x20\x0f\xa0\x33\xce\xe0\x92\x31\x95\x4a\x9c\xc4\x80\x34\xa4\x48\x83\x09\x21\x82\x3e\x83\x30\xf6\x1b\x21\x49\x0c\xb1\xc7\x0c\x9f\x72\x89\x20\xa6\x4c\xc5\x7e\x08\x89\x50\x79\x0c\x12\x03\x22\xb2\xd8\x4b\xb4\x0a\x53\x86\x5c\xc9\x09\x21\x92\xc6\x10\xd4\xbb\x3c\xa6\x24\x6a\x25\x04\xe8\xf2\x9e\x49\x28\x83\x80\x84\xb0\xa6\xa9\xc0\x89\xe7\x79\x2d\x78\xfa\x99\xb2\x29\x4d\x71\xa3\x34\xff\x9d\xda\x33\xa7\xdb\x9f\xcc\x94\x2b\xbf\x06\x7e\xaf\x04\x9c\x0b\x2e\xbc\x20\x68\x49\x85\x5d\x94\x71\x0b\x09\xb4\xc7\xd6\xd1\x30\x78\x9d\x0a\x30\xc1\xc4\x23\x34\xe1\x5f\xb4\x4a\x13\x07\xc2\x23\x9f\x3e\x4d\x08\xd1\x60\x54\xaa\x19\x94\xd7\x40\x86\x89\xe2\x12\xcd\x84\x90\x0c\xf4\x73\x79\x39\x02\x74\xff\x7f\xa3\xc8\x36\xee\x9b\xe0\xa6\xb8\x14\x82\x00\x04\xf7\x35\x4d\x42\x5a\x7e\x65\x1a\xec\xd7\x9e\x4c\xa6\x94\x0e\xb9\x6c\x9a\xac\x0f\x42\x00\x35\x70\x2e\x04\x06\x95\xa6\x11\xec\x15\xce\x0c\x2f\x97\x30\x9a\x50\xc6\x91\xef\x83\x52\x03\xd8\x61\x2a\x85\x76\xa0\x24\xf5\xfd\x12\xea\x41\xae\x48\x54\x38\x20\xb8\xb7\x95\x26\x89\xe9\x6f\xd6\x90\x08\xce\xa8\x81\x21\x4f\x9e\x10\xcd\x57\x22\x35\x08\xfa\xdd\x83\x5a\xa7\x52\x82\x3e\x2a\x84\x13\xab\x96\x41\x90\x98\x29\x91\xc6\xa7\x79\xef\x28\x47\x75\x04\x32\x41\x79\x7c\xb8\xd4\x32\x50\x8e\x0e\xd4\x2a\x4a\x05\x35\x7b\xb3\xa5\x23\xee\x30\x0a\xc8\xa0\x97\xff\x47\x86\x7a\x5f\x17\x49\x13\xb3\x51\x38\x7d\x4d\xa9\xd2\x65\xe5\xf2\x31\xad\xde\x4a\x86\xa5\xfe\xbe\xc2\xaf\x88\x7a\x9d\x43\xa4\x0a\xdf\xd2\x29\x7f\xe0\xb8\xc3\xcc\x41\x11\x29\xdb\xc4\xaf\x58\xa2\x21\xe6\xb4\x8a\xf8\x99\xcb\x90\xcb\xe8\x6c\x75\xdc\xf0\x36\x7d\x28\x01\x23\x85\x51\x09\xb8\x87\xb5\x95\x5c\xd9\x6c\x44\x91\x09\x21\x8d\xca\x7e\x78\x4d\x36\xe9\xf3\xbf\x81\xa1\xa3\xaf\xc1\x9e\xe6\x3d\xba\x91\x06\x7f\x7f\x80\x0b\x4e\xb5\x74\xb3\xea\x1c\x55\x2f\xce\x6f\xf3\x6e\x83\xfa\x51\x9d\xa9\x49\x80\x59\x41\x89\xd2\x58\x37\x0e\x1a\x03\x32\xbb\xb8\xb8\xb8\x70\x08\x90\xea\x08\x70\xd5\xba\x6a\x40\x00\x43\xa5\x0b\x8c\x34\x49\xf6\x89\x3c\x56\x81\x8f\xb2\x53\x0c\xa8\x39\x33\xc7\x1b\x69\x36\x68\xa4\xd9\x59\x8d\x44\x08\xe6\x09\x04\xe4\x56\x85\x60\x45\xf6\xac\x66\x9b\xba\x5d\xf2\x3e\x20\x45\x58\xa7\xe2\x01\x3e\x6c\x00\xaa\x2c\x58\xf5\x95\x01\x99\xf5\x0c\x14\xdb\xc2\xf0\xb5\x81\xe9\x15\x9b\x1d\x0f\xda\x14\x31\x74\x3b\x82\x1d\x21\x4e\x04\x45\x28\x21\x35\xac\xe5\xf0\x48\xa9\xd0\x51\x4d\x0d\x71\x10\x85\x61\x9a\x5a\x07\xd9\xa3\x29\x97\xa0\x4d\xb9\x5a\xb4\xd4\x7b\x55\xc1\xe3\x55\xdc\xbb\xeb\x1b\x3c\x6f\x94\xda\x06\x84\x47\x52\x69\x70\x4b\x2b\xaf\x38\x24\xeb\x35\x97\x1c\xf3\x1d\x36\xdb\x33\x5c\xf6\xae\x5a\x1f\xfe\x96\x72\x0d\xe1\x75\xaa\xb9\x8c\x1e\xd8\x06\xc2\x54\x70\x19\x2d\xdd\xc1\xe5\xe5\xc5\x0b\xb0\xd4\x62\x6a\xee\x2c\xce\x7c\x28\xbd\xfe\x08\x3a\x36\xed\xdb\x5e\x11\x04\x8b\x97\x44\x83\x31\x6d\x33\x57\x2b\xb6\x90\x07\x43\x0a\x0e\x18\xaf\xfa\xa8\x04\x34\xb5\x71\x46\x6e\x15\x2e\x65\xef\x7e\x46\x45\x0a\x3d\x51\xae\x97\x92\xea\xd3\xd0\xe5\x35\x15\x06\xaa\x3b\x3b\x2f\x57\x47\xd8\x41\x35\x8e\xa9\x0c\x77\x67\x7a\xc4\x4f\x8d\xf6\x85\x62\x54\xf8\xcf\x5c\xfa\xb5\xd7\x43\xcd\xb3\x06\x68\x8f\x78\xd9\xfc\xef\xcd\x3f\x63\x15\xc2\x7c\x40\x3d\x8f\x78\xd5\xbc\x3d\x4f\x25\x7f\x09\x7c\xdf\x67\x86\xfb\xed\x50\x9a\x1a\xc5\xb6\xcd\x3d\x1a\x22\x6e\x50\xe7\x8b\x6a\x2f\xb2\x24\xf0\xfd\x8b\xa9\xfb\x17\xec\x88\xbf\x5c\x6f\x7d\xc6\xc3\xf9\x77\xdf\xff\xeb\x97\xcf\x8b\xa7\xdb\xbb\xeb\xc5\xd3\xed\xe5\xcd\xe2\x87\xe6\x1a\x46\x7f\xe6\x02\xe6\x3e\x03\x8d\xc6\x67\x74\xca\x34\xb6\xee\x83\xc6\xe6\x0a\x14\xa6\xbb\x64\x0b\x79\x77\xc5\x16\xf2\xe6\x8a\xc2\x50\x96\x5d\xe6\xdf\x7d\xbf\xba\x59\xdc\x3c\x5d\x3d\x2c\x9f\xae\xef\x97\xbf\x2e\xee\xfb\x90\x4a\x46\xff\xea\x46\xab\x79\xb0\xa3\x6a\xfb\x01\x99\x35\x5d\x53\xb0\x59\x5b\xbf\x49\x27\x3e\x7e\xd6\x2a\x6e\x87\xc8\x9a\x83\x08\xcb\xb6\xa4\xf9\xe9\xd4\xaf\xf6\x4d\xb7\x69\x45\x71\x13\xb8\xf4\x9b\x5a\xeb\x5a\x42\xea\xa1\x79\x5c\xdc\xdf\x2c\x6f\x2f\x1f\x97\x77\xb7\x4f\x5f\xef\xbe\x3c\xad\x2e\x1f\xff\xd9\xc5\x14\x10\x1f\xe3\xc4\x47\xd0\x71\xf9\xa3\x88\x27\x54\xd4\x3b\x6a\xc8\x56\xfd\xa3\xfa\x39\x55\xaf\xe1\x31\x8d\x20\x20\xee\x46\x37\x70\x03\x46\x25\xd5\x79\x7b\xed\x2a\x15\x62\xa5\x04\x67\x79\x40\x96\xeb\x5b\x85\x2b\x0d\x06\xe4\xce\xe1\x8d\xfa\xd1\x89\xff\xba\xc8\x56\x3a\xd4\xe9\xb5\xea\xd4\xdc\xe6\x49\xbb\xfa\x5d\x7c\x0c\xb0\x54\x73\xcc\xaf\xec\x78\xf6\x82\x6d\xfe\xa2\xe1\x9d\x14\xf9\xbd\x52\x2e\x26\x4d\x6e\x10\xe2\x80\xa0\x4e\x77\x5e\x68\x98\xf4\x06\x8c\xb1\x2a\x39\x9f\x8d\xda\xbb\x18\x85\x6e\x6c\x87\xd8\xd2\x20\xb6\x57\xca\xfd\x2e\xba\x7b\xf0\xab\x9c\x74\x89\xb2\x6f\xa7\xe1\xbd\x7d\x89\x48\x23\x2e\x3d\x9b\xe2\x80\x5e\xc8\xf5\x9e\xbd\x18\x27\xbd\xbd\x18\x27\x8d\x1d\x1e\xa1\x3a\x6a\xa1\xf6\xbc\xf2\x17\xac\xbc\xc1\x26\xde\xae\xfa\x94\xbc\x04\x12\x35\x15\x23\xf4\xe5\x62\x85\x86\xa1\xa5\xf4\xf9\x21\x04\xe5\xad\x81\x62\xaa\xc1\x8b\x28\x82\x99\x3f\xaa\x44\x09\x15\xe5\xf3\x96\x83\xec\x3a\x63\x7d\x8e\x1e\xee\x5d\x80\x3c\x06\x95\xe2\xfc\x1f\x71\xeb\x72\x19\x2c\x35\x28\x47\x0e\xb3\x71\x72\x58\xdd\x5d\xbb\xbc\x79\x58\x5d\x5e\xfd\x01\x6e\x68\xa4\x7f\xd5\x5c\x4c\xeb\x7e\x69\xaf\xd0\xb7\x97\xd7\x4d\xed\xdf\x52\x9a\xdb\x89\x6f\xfb\x93\xa9\x3c\xd4\x98\x8b\x82\xec\x47\x5b\x1b\x4e\xca\xf1\xa1\x39\xeb\xa8\x64\x9f\x9d\x3d\xd9\x0f\xca\xdb\x23\xb2\x6f\x2f\x1e\x9d\xca\x4b\x73\xab\xa4\x45\xd3\xc1\xe0\x6e\xfd\x62\x40\x17\x43\x56\x7d\x52\x73\xec\x1c\x6b\x5c\x77\x7a\x34\xba\x10\x88\x13\xcc\xaf\xb9\x0e\x88\x4c\x85\xe8\x92\xef\x1e\xf4\xde\x28\x27\x19\x60\x1a\x5a\x26\x2e\xae\x74\xb0\xd5\x9b\x8b\xbb\x66\x00\xd2\x7f\xfe\xdb\x01\x54\x11\xd2\xf8\x34\x73\x4d\x21\x56\xf2\x8c\xb3\x8c\xad\xc8\xe3\x53\xcc\x91\x73\x4b\x79\xe0\x29\xd0\x3e\x74\x1c\x69\xe0\x7e\xc7\x41\xe4\xcd\xbb\xe9\x10\x6c\x0a\xdd\x50\x49\x23\xd0\x73\x91\xc5\xaf\xb5\xda\x2d\xbd\x47\x9a\xec\x5e\xe1\x3a\xa8\x53\xae\x93\xb6\xd3\x7f\x97\xbb\x2c\xe5\x2f\x57\x3f\xb8\x36\x7c\x76\x40\xdb\x3e\xc0\x06\xfd\x16\xfe\x5d\xda\x73\x83\x14\x5d\x97\x34\xf7\x33\xaa\x7d\xc1\x9f\xfd\x03\xda\xf4\xe3\x9a\x7a\xab\xed\x0a\xb4\x6d\x3a\x68\x04\xf3\x59\x5b\xcd\xff\xf7\x96\xbf\x11\x00\xef\x00\x05\x29\xa6\x66\x9a\xa8\xb0\x21\xed\xcd\x26\x86\x37\x9c\x62\xfe\x54\xc3\x47\xa2\x79\xc6\x05\x44\x10\x76\xea\x7f\xab\x03\xb8\x38\xff\x1c\x52\xa5\xe8\x36\x7d\x06\x01\xe8\x17\xdd\x80\x71\x7f\x6b\x09\x08\xee\x01\x40\xbb\xeb\x29\xf6\x6b\x95\xd0\xc8\x09\x0d\xc8\x67\x1e\x72\x0d\x8e\xf3\x1b\x33\x40\x6d\x17\xbb\xde\x91\xd5\xc8\x68\xd2\x03\x52\x3c\xb7\x3e\x55\xa6\xdd\x3e\x22\x6d\x78\xfc\xb2\x19\x39\x36\x7a\x85\x90\xf5\xf6\x84\x90\x8d\xc8\x31\x79\x5f\x8a\xc9\x47\x91\x0d\x34\x98\xaf\xce\x75\x95\xf1\x46\x32\xf1\x04\x13\xda\xc3\x1c\x9b\x8f\xcf\x87\xdd\x39\xaf\xf4\x60\x55\xa9\x8a\xb8\x4c\x5a\x05\xa1\x1b\x6d\x7b\x28\x7f\xb0\xba\x0e\xce\x91\xed\x55\xc3\x93\xdb\x1b\xfd\xfa\x31\x30\x22\xb9\xb8\x29\x28\xa4\xd6\x5a\x07\xd9\x6c\xfa\xe3\x89\xc3\x52\xf7\xac\x0f\x1f\x71\x5e\x8d\xc0\xa6\xb3\x7b\xbb\x5b\x91\xb0\x3b\xa3\xf9\x2b\xf0\x0e\x4c\xf9\xa0\xb9\x70\xc5\xbe\x89\x66\xa3\x4c\x21\xb9\xc9\xa9\xe3\x8c\x36\x9a\x1b\xc5\x83\x94\x6b\x97\x0d\x4a\xe7\x77\xfa\xaa\x7a\x2f\x61\xd4\x0a\xa7\x00\x79\xaa\x3a\x38\xff\x68\xf9\x7b\xec\x78\x92\x39\xc6\x08\xfe\x30\x34\x83\xb4\x7e\x1c\x96\x36\xc7\x1f\x26\xb6\xc3\xec\xde\x5e\xee\x3e\x70\x44\x75\x1b\xbb\xe3\xe9\x01\x4a\xbc\x41\x3c\xed\x61\xd8\x11\xe1\xed\x0a\x74\x98\x94\x76\x85\x1a\x3d\xbe\x5d\xae\x0e\x4c\x8a\xb2\x9c\x75\xc7\xf5\xf6\x0b\x23\x7e\x36\x7b\x06\xa4\xf5\x8b\x04\x0f\xcb\xeb\xa2\xe5\x3a\xcf\xf4\xde\x70\x4a\x35\x54\x16\xaf\xa6\xdc\x97\x4f\xad\x02\xe2\x9e\xdd\xb8\xa7\xb9\xe1\x52\xae\xd5\x9d\x74\xac\x58\xf3\x65\x09\xff\xaa\xfc\xed\xb3\xbe\x5e\x50\xd1\x57\xbe\x06\x96\x33\x01\x37\x2a\xac\xde\x80\x59\xd5\x2f\x70\xb9\x3f\x17\xc9\x06\x62\xd0\x54\x4c\xfe\x17\x00\x00\xff\xff\x0c\xb2\x52\x9d\xc3\x2a\x00\x00") func deployKubernetes119AlphaLvmPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -199,12 +199,12 @@ func deployKubernetes119AlphaLvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml", size: 10540, mode: os.FileMode(420), modTime: time.Unix(1599471432, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml", size: 10947, mode: os.FileMode(420), modTime: time.Unix(1599822745, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5f\x6f\xe3\xb8\x11\x7f\xf7\xa7\x60\xb7\xf7\x70\x07\x54\x56\x72\x45\x81\x83\x00\x3f\x64\x37\xbe\xad\xd1\x8d\x63\x24\xb9\x7b\x0d\x68\x6a\x2c\xb3\xa6\x48\x1e\x39\xf2\x46\x57\xf4\xbb\x17\xa4\x24\x5b\xff\x2c\xff\xb9\x64\xd3\x73\x1e\x22\x8b\x1c\xce\x8f\x33\xc3\x99\xdf\x48\xfe\x2b\xf9\x0c\x12\x0c\x45\x88\xc9\x57\x8e\x6b\xf2\x21\xa5\x1b\x20\x9b\xcc\xa2\x4a\xf9\xef\xf0\xe1\x6f\x24\x56\x44\x2a\x24\x10\x73\xfc\xcb\x68\x44\x35\xff\x15\x8c\xe5\x4a\x46\x64\x7b\x3d\xda\x70\x19\x47\xe4\x11\xcc\x96\x33\xb8\x61\x4c\x65\x12\x47\x29\x20\x8d\x29\xd2\x68\x44\x88\xa0\x4b\x10\xd6\x5d\x11\xa2\x53\x48\x03\x66\xf9\x98\x4b\x04\x31\x66\x2a\x0d\x63\xd0\x42\xe5\x29\x48\x8c\x48\xcc\x0d\x30\x0c\xb4\x51\x71\xc6\x90\x2b\x39\x22\x44\xd2\x14\xa2\x9d\x60\xc0\x94\x44\xa3\x84\x00\x53\x8e\x59\x4d\x19\x44\x24\x86\x15\xcd\x04\x8e\x82\x20\x68\x20\x34\x4b\xca\xc6\x34\xc3\xb5\x32\xfc\x77\xea\xd6\x1c\x6f\x7e\xb2\x63\xae\xc2\x1d\xf6\x07\x25\xe0\x0d\x11\xc3\x0b\x82\x91\x54\xb8\x49\x5b\xee\x50\x81\x09\xd8\x2a\xe9\xc7\x6f\x32\x01\x36\x1a\x05\x84\x6a\xfe\xd9\xa8\x4c\x7b\x1c\x01\xf9\xf0\x61\x44\x88\x01\xab\x32\xc3\xa0\xbc\x07\x32\xd6\x8a\x4b\xb4\x23\x42\xb6\x60\x96\xe5\xed\x04\xd0\xff\xff\x4a\x91\xad\xfd\x95\xe0\xb6\xb8\x15\x83\x00\x04\x7f\x99\xe9\x98\x96\x97\xcc\x80\xbb\xec\xe8\x64\x4a\x99\x98\xcb\xba\xd5\xba\x20\x04\x50\x0b\x6f\x85\xc0\xa2\x32\x34\x81\x83\xca\x99\xe5\xe5\x14\x46\x35\x65\x1c\xf9\x21\x28\x3b\x00\x7b\x4c\xa5\xd2\x16\x14\xbd\x1b\x2f\xa1\x9e\xe4\x0a\xad\xe2\x1e\xc5\x1d\x51\xaa\xb5\xed\x0a\x1b\xd0\x82\x33\x6a\xa1\xcf\x93\x17\x04\xf4\x27\x91\x59\x04\xf3\x1e\x71\x6d\x32\x29\xc1\x9c\x15\xc5\xda\xed\xcc\x22\x48\xdc\x2a\x91\xa5\x97\x39\xf0\x2c\x5f\xb5\x14\x32\x41\x79\x7a\xba\xd6\x32\x56\xce\x8e\xd5\x2a\x50\x05\xb5\x07\x0f\x4c\x4b\xdd\x69\x59\x60\x0b\x9d\x14\x70\x66\xb4\x77\xf7\x22\xa9\xb6\x6b\x85\xe3\x63\x9b\x2a\x5d\x56\x4e\x1f\xda\xd5\x6b\xe9\x70\x05\xa0\xbb\xe1\x23\xaa\x8e\xa7\x11\xa9\xe2\xd7\x74\xca\x1f\x58\xee\x34\x73\x50\x44\xca\xd6\xe9\x11\x4b\xd4\xd4\x5c\x56\x17\x3f\x72\x19\x73\x99\xbc\x65\x41\xb7\xbc\x99\x41\x94\x80\x81\xf2\xa8\x04\x3c\xc0\xca\x29\xaf\xcc\x36\xb0\x97\x11\x21\xb5\x12\x7f\x7a\x65\xb6\xd9\xf2\xdf\xc0\xd0\x67\xb0\x5e\x7e\xf3\x2d\x68\x49\x2d\x8b\xbf\x8f\x17\x2e\x35\x76\xbd\xfc\x9c\x55\x35\xde\xde\xec\x6d\xbe\xfa\x8e\x44\xd5\x6a\x60\x4e\x97\x56\x06\x77\x24\xc2\x60\x44\xae\xaf\xae\xae\xae\x3c\x08\xa4\x26\x01\x5c\x34\xee\x5a\x10\xc0\x50\x99\x02\x26\xd5\xfa\x90\xca\x0b\xf6\xf0\x8e\xd6\x4a\x01\x0d\x67\xf6\x7c\x53\x5d\xf7\x9a\xea\xfa\xad\x4d\x45\x08\xe6\x1a\x22\x32\x57\x31\x38\xad\x1d\xdb\x39\xa6\xb7\x3f\xcb\x8f\x48\x11\x56\x99\x78\x84\xf7\xec\x8d\x2a\x3b\x56\x7c\x33\x22\xd7\x1d\x33\xa5\xae\x5a\x7c\xa9\xc1\x3a\x62\xb9\x8b\x70\xdb\x22\x9e\xe6\x03\xf0\x11\x52\x2d\x28\x42\x89\xaa\x66\x33\xf7\x11\x0d\x80\x47\x21\x5e\x04\xf2\xa0\xe0\x57\x58\xae\x95\xda\x44\x84\x27\x52\x19\xf0\x53\x2b\xd3\x7a\x30\xab\x15\x97\x1c\xf3\x3d\x3c\xc7\x06\x6e\x3a\x77\x9d\x23\x7e\xcb\xb8\x81\xf8\x36\x33\x5c\x26\x8f\x6c\x0d\x71\x26\xb8\x4c\x66\x7e\xe1\xf2\xf6\xf4\x05\x58\xe6\x30\xd5\x25\x8b\x35\x1f\x4b\xd7\x3d\x81\x49\x6d\x73\x38\x28\x3c\x39\x7d\xd1\x06\xac\x0b\xc9\xd6\xb8\x9b\xb1\x81\x3c\xea\xdb\x60\x8f\xfd\xaa\x8f\xd2\x60\xa8\x0b\x16\x32\x57\x38\x93\x9d\xf1\x2d\x15\x19\x74\x54\x79\x96\x24\xd5\x87\xbe\xdb\x2b\x2a\x2c\x54\x23\x4e\x33\xe5\x12\xcc\x6e\x09\xd7\x85\xa6\x29\x95\xf1\x7e\xcd\x80\x84\x99\x35\xa1\x50\x8c\x8a\x70\xc9\x65\xb8\x73\x7c\x6c\xf8\xb6\x06\x3a\x20\xc1\x76\xf2\xf7\xfa\xd7\x54\xc5\x30\xe9\xd9\x5e\x40\x82\xaa\x99\x9e\x64\x92\xbf\x44\x61\x18\x32\xcb\xc3\x66\x34\x8d\xad\x62\x9b\xba\x8c\x81\x84\x5b\x34\xf9\xb4\x92\x45\xa6\xa3\x30\xbc\x1a\xfb\xbf\x68\x9f\xc9\x2b\xf5\x45\x96\xfb\xe2\xbb\x8f\x49\xb4\x4f\x5f\xe5\xb8\xf3\x29\x8f\x27\xdf\x7d\xff\xaf\x5f\x3e\x4e\x9f\xe7\xf7\xb7\xd3\xe7\xf9\xcd\xdd\xf4\x87\xfa\x1c\x46\x7f\xe6\x02\x26\x21\x03\x83\x36\x64\x74\xcc\x0c\x36\xc6\xc1\x60\x7d\x06\x0a\xdb\x9e\xb2\x81\xbc\x3d\x63\x03\x79\x7d\x46\x61\x48\x97\x42\x26\xdf\x7d\xbf\xb8\x9b\xde\x3d\x7f\x7a\x9c\x3d\xdf\x3e\xcc\x7e\x9d\x3e\xb4\x20\x81\xdc\xd6\x5d\x53\xa4\xa4\x26\xfe\x51\x2b\x3e\x7e\x36\x2a\x6d\x86\xc8\x8a\x83\x88\x4b\xaa\x51\xff\xb4\xaa\x51\x73\xd0\x0b\x2d\x28\xae\x23\x7f\xfc\xc6\xce\x7a\x2e\xa5\x74\xd0\x3c\x4d\x1f\xee\x66\xf3\x9b\xa7\xd9\xfd\xfc\xf9\xcb\xfd\xe7\xe7\xc5\xcd\xd3\x3f\xdb\x98\x22\x12\x62\xaa\x43\x04\x93\x96\x4f\x3c\x02\xa1\x92\xce\x52\x7d\xb6\xe8\x2e\xd5\x3d\x53\xbb\x39\x3c\xa5\x09\x44\xc4\x0f\xb4\x03\x37\x62\x54\x52\x93\x37\xe7\x2e\x32\x21\x16\x4a\x70\x96\x47\x64\xb6\x9a\x2b\x5c\x18\xb0\x20\xf7\x0e\xad\x15\x81\x56\xfc\x5b\x60\x99\xe1\x98\x7f\x72\x2d\xd3\x0b\x36\x33\x0f\x8d\xef\xa5\xc8\x1f\x94\xf2\xd1\x62\x73\x8b\x90\x46\x04\x4d\xb6\xb7\x5f\xcd\x18\x77\x60\xad\x03\xe3\xad\x3d\x68\xa9\xa2\x3d\xb9\x73\x7c\xcd\xd6\x03\x23\x75\x77\x4a\x79\x1f\x77\x35\x34\xc5\x16\xaa\xd3\xe4\x43\xf8\x90\xa4\xe5\x1d\x39\x2d\xb2\x84\xcb\xc0\x1d\x4e\xc0\x20\xe6\xe6\x80\x2c\xa6\xba\x23\x8b\xa9\xae\x49\x04\x84\x9a\xa4\x81\xba\x9d\x40\xbc\xb7\x68\x1c\xbb\xa4\x3a\x39\x25\x45\x04\x2b\xa0\x98\x19\x08\x12\x8a\x60\x27\x4f\x4a\x2b\xa1\x92\x7c\xd2\x30\xb4\x9b\x67\x5d\x6a\xc0\x00\x0f\x4e\x40\x9e\x82\xca\x70\xf2\x8f\x4e\x30\xfd\x96\xd1\xdc\xb5\x0e\x9b\x9f\x6c\x85\xa8\xc6\xae\xa3\xed\x8f\x2e\x1b\x5d\x14\x55\x7d\x6c\xfd\xd5\xc2\xeb\xa4\x48\x39\xc3\xdf\x07\xf1\x98\x4c\xde\xd8\xb9\x92\x0e\x4d\x0b\x83\x1f\xfa\xc5\x82\x29\x18\xf6\x6e\xa5\x7a\xdb\x31\x44\x53\xf6\xfb\xa8\x55\x2c\x48\x35\xe6\xb7\xdc\x44\x44\x66\x42\xb4\x0f\xea\x01\xf4\xc1\xe0\x29\xb0\xc0\x0c\x34\x4c\x5c\xdc\x69\x61\xdb\x09\x17\xa3\xb6\x07\xd2\x7f\xfe\x3b\xea\x3f\x02\xc3\x0c\xf6\x96\x42\xaa\xe4\xdb\xf2\x57\x97\xc0\x87\x99\xeb\x99\x5c\xb5\x5c\xf0\x42\x74\x7f\x84\x82\xd6\x34\x7f\x5b\xf2\xf9\xea\x0c\x2a\x06\x77\x14\xee\xa8\xa4\x09\x98\x49\x81\xf5\x18\xc3\x6a\xec\x7e\x80\x5b\x75\xb2\xe5\x49\x04\x68\x77\xfe\x5a\xb4\xab\x94\x5a\xdc\xdf\x3e\xcf\x16\x3f\x78\xf6\x75\x7d\x02\x5b\xeb\x39\xd8\x5d\xe6\xf6\x4d\x58\x97\x75\x5d\xa2\x4b\x7c\x93\x70\x4b\x4d\x28\xf8\x32\x3c\xc6\xbe\xce\xe3\x6a\x01\x09\xdc\x6e\x17\x60\x18\x48\xa4\x09\x4c\xae\x6b\xdb\xfc\x13\x30\xb9\x9a\x83\xbf\x01\x14\xa4\x98\xd9\xb1\x56\x71\x4d\xdb\xab\x11\xc1\x57\x24\xa7\xff\x27\x9c\x52\x1b\xbe\xe5\x02\x12\x88\x5b\x45\xb6\x51\x66\xaf\xde\x9e\x5e\x56\x87\x67\x93\x2d\x41\x00\x86\x45\xc9\xb5\xfe\xbb\x91\x80\xe0\x1f\xb4\x36\xa9\x45\x21\x6f\x94\xa6\x89\x57\x1a\x91\x8f\xbc\x48\x76\x5c\x49\x2a\x3a\x24\xc4\xcf\xf7\x69\x64\x80\x71\x76\x80\x14\x6f\x09\x2f\xd5\xe9\xc4\x07\xb4\xf5\xb3\x6a\x77\x96\x86\x18\x75\x0c\xdb\x8e\x4c\x0c\xdb\x01\x3d\x36\xef\x6a\xb1\xf9\x20\xb2\x1e\x16\x77\x94\xae\x57\xc6\x1b\x38\x43\x17\x98\xd0\x2d\xe6\xf3\xec\x79\xb4\xbf\xf4\x60\x55\x43\x8a\xb8\xd4\x8d\x54\xdd\x8e\xb6\x03\xc9\xb8\xb7\xee\xf5\xb6\x15\xcd\x59\xbd\xd9\xf9\xb5\xda\xd1\x9e\x0e\xc2\xc7\x4d\x71\xf8\x77\xbb\x36\xd1\xf6\x7a\xfc\xe3\x85\xbd\x44\x7b\xad\x77\xef\x23\x8e\x46\x60\xdd\xd9\x87\xba\xd5\x22\x12\xf6\x6b\xd4\x1f\xcb\xed\xc1\x94\xef\xf4\x0a\x57\x1c\x6a\x1b\xd6\xca\x16\x9a\xeb\x39\x75\x38\xa3\x0d\x9e\x8d\xe2\x09\xf5\xad\x3f\x0d\xca\xe4\xf7\xe6\x53\xf5\x0a\x78\xd0\x0a\x97\x00\x79\xae\xb8\x55\x78\xb6\xfe\x03\x76\xbc\xc8\x1c\x43\x09\xfe\x34\x34\xbd\x69\xfd\x3c\x2c\xcd\x1c\x7f\x9a\xda\x56\x66\x0f\x0e\xe6\xee\x13\xfb\x40\x2f\xd8\xee\x01\x4f\xd8\xc4\x2b\xc4\xd3\x81\x0c\x3b\xa0\xbc\x59\x81\x4e\xd3\xd2\xac\x50\x83\xcb\x37\xcb\xd5\x89\x87\xa2\x2c\x67\xed\x9e\xb8\xf9\x6e\x3e\xdc\x5e\x2f\x01\xe9\xee\x85\xed\xe3\xec\xb6\x20\x4b\x6f\xd6\x22\xd7\xfc\x52\x35\x7d\xc5\x0f\x01\x1e\xca\x37\x09\x11\xf1\xcf\xd3\xfd\xcb\xb2\x78\x26\x57\xea\x5e\xfa\xc4\xb8\x4b\x99\x45\xd6\xf9\xc2\x57\xc0\x72\x26\xe0\x4e\xc5\xd5\xef\x0a\x16\xbb\x9f\xc5\xf8\xaf\x53\xbd\x86\x14\x0c\x15\xa3\xff\x05\x00\x00\xff\xff\x56\x33\x4f\x09\x22\x28\x00\x00") +var _deployKubernetes119DirectPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\xdd\x6f\xe3\xb8\x11\x7f\xf7\x5f\xc1\xa6\xf7\x70\x07\x54\x56\x7c\x45\x81\x83\x00\x3f\xe4\x36\xbe\xad\xd1\x4d\x62\x24\xb9\x7b\x0d\x18\x6a\x2c\xb3\xa6\x48\x1e\x39\xf2\x46\x57\xf4\x7f\x2f\x48\x7d\x58\x5f\x56\x6c\x37\xd9\xa0\xf5\x3e\xac\x22\x71\x38\xbf\xf9\xe0\xcc\x6f\x2c\xff\x99\x7c\x06\x09\x86\x22\xc4\xe4\x2b\xc7\x0d\xb9\x48\xe9\x16\xc8\x36\xb3\xa8\x52\xfe\x07\x5c\xfc\x85\xc4\x8a\x48\x85\x04\x62\x8e\x7f\x9a\x4c\xa8\xe6\xbf\x81\xb1\x5c\xc9\x88\xec\x66\x93\x2d\x97\x71\x44\x1e\xc0\xec\x38\x83\x2b\xc6\x54\x26\x71\x92\x02\xd2\x98\x22\x8d\x26\x84\x08\xfa\x0c\xc2\xba\x2b\x42\x74\x0a\x69\xc0\x2c\x9f\x72\x89\x20\xa6\x4c\xa5\x61\x0c\x5a\xa8\x3c\x05\x89\x11\x89\xb9\x01\x86\x81\x36\x2a\xce\x18\x72\x25\x27\x84\x48\x9a\x42\x54\x0b\x06\x4c\x49\x34\x4a\x08\x30\xe5\x33\xab\x29\x83\x88\xc4\xb0\xa6\x99\xc0\x49\x10\x04\x2d\x84\xe6\x99\xb2\x29\xcd\x70\xa3\x0c\xff\x83\xba\x3d\xa7\xdb\x9f\xec\x94\xab\xb0\xc6\x7e\xaf\x04\xbc\x23\x62\x78\x41\x30\x92\x0a\xb7\x68\xc7\x1d\x2a\x30\x01\x5b\x27\xc3\xf8\x4d\x26\xc0\x46\x93\x80\x50\xcd\x3f\x1b\x95\x69\x8f\x23\x20\x17\x17\x13\x42\x0c\x58\x95\x19\x06\xe5\x3d\x90\xb1\x56\x5c\xa2\x9d\x10\xb2\x03\xf3\x5c\xde\x4e\x00\xfd\xff\x5f\x29\xb2\x8d\xbf\x12\xdc\x16\xb7\x62\x10\x80\xe0\x2f\x33\x1d\xd3\xf2\x92\x19\x70\x97\x3d\x9d\x4c\x29\x13\x73\xd9\xf4\x5a\x1f\x84\x00\x6a\xe1\xbd\x10\x58\x54\x86\x26\x70\x50\x39\xb3\xbc\x5c\xc2\xa8\xa6\x8c\x23\x3f\x04\xa5\x06\xb0\xc7\x54\x2a\xed\x40\xd1\xf5\xf3\x12\xea\x51\xa1\xd0\x2a\x1e\x50\xdc\x13\xa5\x5a\xdb\xbe\xb0\x01\x2d\x38\xa3\x16\x86\x22\x79\x46\x42\x7f\x12\x99\x45\x30\x1f\x91\xd7\x26\x93\x12\xcc\x49\x59\xac\x9d\x65\x16\x41\xe2\x4e\x89\x2c\x3d\x2f\x80\x27\xc5\xaa\xa3\x90\x09\xca\xd3\xe3\xb5\x96\xb9\x72\x72\xae\x56\x89\x2a\xa8\x3d\x78\x60\x3a\xea\x8e\xab\x02\x3b\xe8\x95\x80\x13\xb3\xbd\x6f\x8b\xa4\xda\x6e\x14\x4e\x5f\x33\xaa\x0c\x59\xb9\x7c\xcc\xaa\xb7\xd2\xe1\x1a\x40\xdf\xe0\x57\x54\xbd\x5e\x46\xa4\x8a\xdf\x32\x28\xff\xc5\x76\xc7\xb9\x83\x22\x52\xb6\x49\x5f\xf1\x44\x43\xcd\x79\x7d\xf1\x67\x2e\x63\x2e\x93\xf7\x6c\xe8\x96\xb7\x2b\x88\x12\x30\xd2\x1e\x95\x80\x7b\x58\x3b\xe5\x95\xdb\x46\x6c\x99\x10\xd2\x68\xf1\xc7\x77\x66\x9b\x3d\xff\x13\x18\xfa\x0a\x36\xc8\x6f\xbe\x05\x2d\x69\x54\xf1\x8f\x89\xc2\xb9\xce\x6e\xb6\x9f\x93\xba\xc6\xfb\xbb\xbd\xcb\x57\x3f\x90\xa8\x5a\x0d\xcc\xe9\xd2\xca\x60\x4d\x22\x0c\x46\x64\x76\x79\x79\x79\xe9\x41\x20\x35\x09\xe0\xaa\x75\xd7\x82\x00\x86\xca\x14\x30\xa9\xd6\x87\x54\x9e\x61\xc3\x07\x7a\x2b\x05\x34\x9c\xd9\xd3\x5d\x35\x1b\x74\xd5\xec\xbd\x5d\x45\x08\xe6\x1a\x22\x72\xab\x62\x70\x5a\x7b\xbe\x73\x4c\x6f\x7f\x96\x1f\x90\x22\xac\x33\xf1\x00\x1f\x39\x1b\x55\x7e\xac\xf8\x66\x44\x66\x3d\x37\xa5\xae\x5b\x7c\x69\xc0\x7a\xc5\x73\x67\xe1\xb6\x45\x3e\xdd\x8e\xc0\x47\x48\xb5\xa0\x08\x25\xaa\x86\xcf\x3c\x24\x29\x15\xfa\xe2\x53\xa3\x1c\x04\x62\x99\xa1\x2e\x4c\x6e\x6b\xca\x25\x18\x5b\xae\x16\x2d\x0b\x5f\xb5\xf1\x2c\x2b\x0f\x0a\x7e\x85\xe7\x8d\x52\xdb\x88\xf0\x44\x2a\x03\x7e\x69\x15\x1b\x0f\x66\xbd\xe6\x92\x63\xbe\x87\xe7\xe8\xc4\x55\xef\xae\x8b\xe4\xef\x19\x37\x10\x5f\x67\x86\xcb\xe4\x81\x6d\x20\xce\x04\x97\xc9\xd2\x6f\x5c\xde\x5e\xbc\x00\xcb\x1c\xa6\xa6\x64\xb1\xe7\x43\x19\xfb\x47\x30\xa9\x6d\x3f\x0e\x8a\x54\x58\xbc\x68\x03\xd6\xb6\x3d\x5d\xad\xd8\x42\x1e\x0d\x19\x38\xe0\xbf\xea\xa3\x34\x18\xea\xb2\x8d\xdc\x2a\x5c\xca\xde\xf3\x1d\x15\x19\xf4\x54\x79\x9a\x25\xd5\xc5\xd0\xed\x35\x15\x16\xaa\x27\xfb\x40\x57\x5b\xb8\x31\x36\x4d\xa9\x8c\xf7\x7b\x06\x24\xcc\xac\x09\x85\x62\x54\x84\xcf\x5c\x86\x75\xe0\x63\xc3\x77\x0d\xd0\x01\x09\x76\xf3\xbf\x36\xff\x4c\x55\x0c\xf3\x01\xf3\x02\x12\x54\xd3\xf8\x3c\x93\xfc\x25\x0a\xc3\x90\x59\x1e\xb6\xb3\x69\x6a\x15\xdb\x36\x65\x0c\x24\xdc\xa2\xc9\x17\x95\x2c\x32\x1d\x85\xe1\xe5\xd4\xff\x8b\xf6\xad\xa0\x5c\xef\x62\xc6\xe3\xf9\x77\xdf\xff\xe3\xd7\x9f\x17\x4f\xb7\x77\xd7\x8b\xa7\xdb\xab\x9b\xc5\x0f\xcd\x35\x8c\xfe\xc2\x05\xcc\x43\x06\x06\x6d\xc8\xe8\x94\x19\x6c\x3d\x07\x83\xcd\x15\x28\x6c\x77\xc9\x16\xf2\xee\x8a\x2d\xe4\xcd\x15\x85\xa3\x5c\x8d\x99\x7f\xf7\xfd\xea\x66\x71\xf3\xf4\xe9\x61\xf9\x74\x7d\xbf\xfc\x6d\x71\xdf\x87\x54\x56\xf7\x2f\x7e\xea\x9a\x47\xfb\xb2\xed\x3e\x20\x77\xcd\xd0\x14\x35\xad\x6d\xdf\xa4\x93\x1f\xbf\x18\x95\xb6\x53\x64\xcd\x41\xc4\x25\x57\x69\x7e\x3a\xed\xac\xfd\xd0\x0b\xad\x28\x6e\x22\x7f\xfc\xa6\xce\xbb\xae\x26\xf5\xd0\x3c\x2e\xee\x6f\x96\xb7\x57\x8f\xcb\xbb\xdb\xa7\x2f\x77\x9f\x9f\x56\x57\x8f\x7f\xef\x62\x8a\x48\x88\xa9\x0e\x11\x4c\x5a\x7e\x65\x12\x08\x95\xf4\xb6\x1a\xf2\x55\x7f\xab\xfe\x99\xaa\xd7\xf0\x94\x26\x10\x11\xff\xa0\x9b\xb8\x11\xa3\x92\x9a\xbc\xbd\x76\x95\x09\xb1\x52\x82\xb3\x3c\x22\xcb\xf5\xad\xc2\x95\x01\x0b\x72\x1f\xf0\x46\x17\xe9\xe4\x7f\xdd\x70\x2b\x1b\xea\xe3\xb5\xea\xf4\xdf\xe6\x4e\xfb\x5e\x5e\x7c\x2c\xb0\xcc\x70\xcc\x3f\xb9\xc9\xed\x05\xdb\xf5\x8b\xc6\x77\x52\xe4\xf7\x4a\xf9\x9c\xb4\xb9\x45\x48\x23\x82\x26\xdb\x47\xa1\xe1\xd2\x1b\xb0\xd6\x99\xe4\x63\x36\xea\xef\x62\x4a\xba\x71\xb4\xb1\x65\x41\xea\xee\x94\xf2\x3e\xbb\x7b\xf0\xab\x33\xe9\x0f\xca\x21\x49\xcb\x7b\x72\x5a\x64\x09\x97\x81\x3b\xe2\x80\x41\xcc\xcd\x01\x59\x4c\x75\x4f\x16\x53\xdd\x90\x08\x08\x35\x49\x0b\x75\xb7\x0c\xf9\x98\xd3\x38\x76\xa5\x79\x7e\x4c\xa1\x09\xd6\x40\x31\x33\x10\x24\x14\xc1\xce\x1f\x95\x56\x42\x25\xf9\xbc\xe5\x68\xb7\xce\xba\xd8\x61\x80\x07\x17\x20\x4f\x41\x65\x38\xff\x5b\xda\xba\x5d\x06\xbd\x06\xe5\x0f\xf9\xac\x9b\xb5\xbf\x67\x34\x77\x43\xce\xf6\x27\x5b\x81\x6e\xcc\x01\xd1\xee\x47\x57\xf6\xce\x4a\xdf\xa1\xb9\xe2\xa4\x3c\x9e\xbd\x7b\x1e\x1f\x95\x92\x27\x24\xd6\x41\x3c\x26\x93\x57\xf6\x56\x49\x87\xa6\x83\xc1\x3f\xfa\xd5\x82\x29\x26\x8a\x7a\xa7\xe6\x98\x35\x46\xcb\xf6\x76\x34\x1a\x2c\xa4\x1a\xf3\x6b\x6e\x22\x22\x33\x21\xba\x75\xe5\x00\xfa\x60\xf4\xb8\x59\x60\x06\x5a\x2e\x2e\xee\x74\xb0\xd5\xc2\xc5\x53\x3b\x00\xe9\x5f\xff\xee\x00\xaa\xce\xda\x38\x63\xbf\xa6\x90\x2a\xf9\xbe\x7c\xdd\xf5\x9b\x71\xa6\x7e\x22\x37\x2f\x37\x3c\x13\xdd\x87\x52\xee\x06\xf4\x6f\x4b\xb6\xdf\x9c\x31\xc6\xe0\xce\xd2\x0d\x95\x34\x01\x33\x2f\xb0\xbe\xc6\x28\x5b\xd6\x8f\x70\xc9\x5e\x5d\x3f\x8a\x10\xd6\x07\xb8\x43\x33\x4b\xa9\xd5\xdd\xf5\xd3\x72\xf5\x83\x67\x9b\xb3\x23\xd8\xe9\x40\x65\xe8\x33\xd5\x6f\xc2\x42\xad\x1b\xab\x5d\xe5\x9c\x87\x3b\x6a\x42\xc1\x9f\xc3\x23\xd8\xe8\x69\xdc\xd5\x59\xbb\x02\xc3\x40\x22\x4d\x60\x3e\x6b\x9b\xf9\xbf\xce\x6c\x1b\x09\xf0\x0d\xa0\x20\xc5\xcc\x4e\xb5\x8a\x1b\xda\xde\x8c\x18\xbf\x21\x59\xff\xbf\xe2\xd8\xda\xf0\x1d\x17\x90\x40\xdc\xe1\x02\x2d\x36\x70\xf9\xfe\x74\xbb\x3a\xa2\xdb\xec\x19\x04\x60\x58\x30\x03\xeb\xff\x36\x12\x10\xfc\xf7\xdf\x6d\x06\x54\xc8\x1b\xa5\x69\xe2\x95\x46\xe4\x67\x5e\x94\x54\xae\x24\x15\x7d\xbf\xb8\xf5\xbe\x58\x8d\x30\xf0\x1e\x90\xe2\xe5\xed\xb9\x3a\x9d\xf8\x88\xb6\xe1\x29\xc3\x9d\xc8\xb1\x09\x23\x86\x5d\x4f\x26\x86\xdd\x88\x1e\x9b\xf7\xb5\xd8\x7c\x14\xd9\x00\xd9\x7c\x75\x7c\xa9\x9c\x37\x72\x12\xcf\x70\xa1\xdb\xcc\x57\xf3\xd3\xc6\xa0\x32\x82\x55\xa7\x2a\xf2\x52\xb7\x1a\x42\x37\xdb\x0e\x94\xfc\xc1\xee\x3a\x38\x66\xb5\x57\x0d\xd6\xf8\xb7\x1a\xf2\x07\xc6\x25\x9f\x37\x45\x09\xa9\xad\x36\xd1\x6e\x36\xfd\xf1\xcc\xc1\xa9\xbb\xd7\x87\x8f\x3b\xaf\x66\x60\x33\xd8\x87\xa6\xf7\x22\x13\xf6\x7b\x34\xbf\xec\xdc\x83\x29\x5f\xb5\x16\xa1\x38\x34\xdd\x6c\x94\x2d\x34\x37\x6b\xea\x78\x45\x1b\x3d\x1b\xc5\x8b\x83\x6b\x7f\x1a\x94\xc9\xef\xcc\xa7\xea\xcd\xfc\xa8\x17\xce\x01\xf2\x54\x31\xb8\xf0\x64\xfd\x07\xfc\x78\x96\x3b\xc6\x0a\xfc\x71\x68\x06\xcb\xfa\x69\x58\xda\x35\xfe\x38\xb5\x9d\xca\x1e\x1c\xac\xdd\x47\x8e\xab\x5e\xb0\x3b\xaa\x1e\x61\xc4\x1b\xe4\xd3\x81\x0a\x3b\xa2\xbc\xdd\x81\x8e\xd3\xd2\xee\x50\xa3\xdb\xb7\xdb\xd5\x91\x87\xa2\x6c\x67\xdd\xd1\xbd\xfd\x93\x89\x70\x37\x7b\x06\xa4\xf5\x7b\xf4\x87\xe5\x75\x41\xb9\xde\x6d\x92\x6f\xc4\xa5\x1a\x2d\x8b\xdf\x67\xdc\x97\xef\x67\x22\xe2\xdf\x52\xf8\x77\x98\xf1\x52\xae\xd5\x9d\xf4\x85\xb1\x2e\x99\x45\xd5\xf9\xc2\xd7\xc0\x72\x26\xe0\x46\xc5\xd5\xcf\x3d\x56\xf5\xaf\x95\xfc\x9f\x0b\xbd\x81\x14\x0c\x15\x93\xff\x04\x00\x00\xff\xff\x27\xe4\x4c\xed\xb9\x29\x00\x00") func deployKubernetes119DirectPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -219,12 +219,12 @@ func deployKubernetes119DirectPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19/direct/pmem-csi.yaml", size: 10274, mode: os.FileMode(420), modTime: time.Unix(1599471428, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19/direct/pmem-csi.yaml", size: 10681, mode: os.FileMode(420), modTime: time.Unix(1599822741, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _deployKubernetes119LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xd3\x7d\xd8\x05\x2a\x2b\xde\xa2\xc0\x42\x80\x1f\x32\x13\xef\xd4\xe8\xc4\x31\x92\xec\xbe\x06\x34\x75\x2c\xb3\xa6\x48\x2e\x79\xa4\x89\xb6\xe8\x7f\x2f\xa8\x8b\xad\x9b\xe5\xcb\xe6\xd2\xcd\x3c\x8c\x23\xf2\xf0\x7c\x3c\xd7\xef\x58\xf9\x2b\xf9\x02\x12\x0c\x45\x08\xc9\x37\x8e\x1b\x72\x15\xd3\x2d\x90\x6d\x62\x51\xc5\xfc\x77\xb8\xfa\x1b\x09\x15\x91\x0a\x09\x84\x1c\xff\x32\x1a\x51\xcd\x7f\x05\x63\xb9\x92\x01\x49\x27\xa3\x2d\x97\x61\x40\x1e\xc1\xa4\x9c\xc1\x0d\x63\x2a\x91\x38\x8a\x01\x69\x48\x91\x06\x23\x42\x04\x5d\x81\xb0\xee\x13\x21\x3a\x86\xd8\x63\x96\x8f\xb9\x44\x10\x63\xa6\x62\x3f\x04\x2d\x54\x16\x83\xc4\x80\x88\x34\xf6\xb4\x51\x61\xc2\x90\x2b\x39\x22\x44\xd2\x18\x82\x9d\x94\xc7\x94\x44\xa3\x84\x00\x53\xae\x59\x4d\x19\x04\x24\x84\x35\x4d\x04\x8e\x3c\xcf\x6b\xc0\x33\x2b\xca\xc6\x34\xc1\x8d\x32\xfc\x77\xea\xce\x1c\x6f\x7f\xb2\x63\xae\xfc\x1d\xf0\x07\x25\xe0\xad\xe0\xc2\x0b\x82\x91\x54\xb8\x4d\x29\x77\x90\xc0\x78\x6c\x1d\xf5\x83\x37\x89\x00\x1b\x8c\x3c\x42\x35\xff\x62\x54\xa2\x73\x10\x1e\xb9\xba\x1a\x11\x62\xc0\xaa\xc4\x30\x28\x9f\x81\x0c\xb5\xe2\x12\xed\x88\x90\x14\xcc\xaa\x7c\x1c\x01\xe6\xff\x7f\xa3\xc8\x36\xf9\x27\xc1\x6d\xf1\x28\x04\x01\x08\xf9\xc7\x44\x87\xb4\xfc\xc8\x0c\xb8\x8f\x1d\x9d\x4c\x29\x13\x72\x59\x37\x59\x17\x84\x00\x6a\xe1\xad\x10\x58\x54\x86\x46\x70\x50\x39\xb3\xbc\xdc\xc2\xa8\xa6\x8c\x23\x3f\x04\x65\x07\x60\x8f\xa9\x54\xda\x82\xa2\x77\xeb\x25\xd4\x93\x5c\xa1\x55\xd8\xa3\xb8\x23\x4a\xb5\xb6\x5d\x61\x03\x5a\x70\x46\x2d\xf4\x79\xf2\x82\x68\xfe\x2c\x12\x8b\x60\xde\x3d\xa8\x4d\x22\x25\x98\xb3\x42\x58\xbb\x6b\x59\x04\x89\xa9\x12\x49\x7c\x99\xf7\xce\x72\x54\x4b\x21\x13\x94\xc7\xa7\x6b\x2d\x03\xe5\xec\x40\xad\xa2\x54\x50\x7b\x30\x5b\x5a\xea\x4e\x2b\x01\x29\x74\xf2\xff\xcc\x50\xef\xde\x45\x52\x6d\x37\x0a\xc7\xc7\x2e\x55\xba\xac\xdc\x3e\x74\xab\xd7\xd2\xe1\x4a\x7f\xf7\xc2\x47\x54\x1d\xaf\x21\x52\x85\xaf\xe9\x94\x3f\x70\xdc\x69\xe6\xa0\x88\x94\x6d\xe2\x23\x96\xa8\xa9\xb9\xac\x23\x7e\xe2\x32\xe4\x32\x7a\xb3\x3e\x6e\x79\xb3\x7c\x28\x01\x03\x8d\x51\x09\x78\x80\xb5\xd3\x5c\xd9\x6c\xe0\x22\x23\x42\x6a\x9d\xfd\xf4\x9e\x6c\x93\xd5\xbf\x81\x61\x5e\xbe\x7a\x39\xcd\x7b\xb0\x91\x5a\xfd\xfe\x00\x17\x5c\x6a\xe9\x7a\xd7\x39\xab\x5f\xbc\xbd\xcd\xdb\x04\xf5\xa3\x98\xa9\xd5\xc0\x9c\x22\xad\x0c\xee\x88\x83\xc1\x80\x4c\xae\xaf\xaf\xaf\x73\x04\x48\x4d\x04\xb8\x6c\x3c\xb5\x20\x80\xa1\x32\x05\x46\xaa\xf5\x21\x95\xe7\x5e\xe0\xa3\xec\x14\x03\x1a\xce\xec\xf9\x46\x9a\xf4\x1a\x69\xf2\xa6\x46\x22\x04\x33\x0d\x01\x59\xa8\x10\x9c\xca\x8e\xd5\x1c\xa9\xdb\x27\xef\x23\x52\x84\x75\x22\x1e\xe1\xc3\x06\xa0\xca\x82\x15\xaf\x0c\xc8\xa4\x63\xa0\xd8\x35\x86\xaf\x35\x4c\x47\x6c\x76\x3e\x68\x5b\xc4\xd0\x62\x00\x3b\x42\xac\x05\x45\x28\x21\xd5\xac\xe5\x7e\x44\x03\xdd\x51\x7c\xe7\x23\x3c\x28\xf5\x0d\x56\x1b\xa5\xb6\x01\xe1\x91\x54\x06\xf2\xad\x95\x51\x73\x24\xeb\x35\x97\x1c\xb3\x3d\x36\xd7\xf2\x6f\x3a\x4f\x9d\x0b\x7e\x4b\xb8\x81\xf0\x36\x31\x5c\x46\x8f\x6c\x03\x61\x22\xb8\x8c\xe6\xf9\xc1\xe5\xe3\xd9\x0b\xb0\xc4\x61\xaa\x4b\x16\x67\x3e\x96\x4e\x7b\x02\x13\xdb\xe6\xb2\x57\xf8\x70\xf6\xa2\x0d\x58\x17\x89\xad\x75\xb7\x63\x0b\x59\xd0\x77\xc1\x1e\xe3\x55\x3f\x4a\x83\xa1\x2e\x4c\xc8\x42\xe1\x5c\x76\xd6\x53\x2a\x12\xe8\xa8\xca\xa9\x90\x54\x57\x7d\x8f\xd7\x54\x58\xa8\x56\x9c\x66\xca\x25\x98\xdd\x11\x6e\xce\x8c\x63\x2a\xc3\xfd\x99\x1e\xf1\x13\x6b\x7c\xa1\x18\x15\xfe\x8a\x4b\x7f\xe7\xf5\xd0\xf0\xb4\x06\xda\x23\x5e\x3a\xfd\x7b\xfd\xd7\x58\x85\x30\xed\xb9\x9e\x47\xbc\x6a\x5c\x9e\x26\x92\xbf\x04\xbe\xef\x33\xcb\xfd\x66\x28\x8d\xad\x62\xdb\xba\x8c\x81\x88\x5b\x34\xd9\xac\x92\x45\xa6\x03\xdf\xbf\x1e\xe7\xff\x82\x7d\xdd\xae\xd4\x17\x95\xed\x6b\x3e\x62\x4c\x83\x7d\xc9\x2a\xd7\x9d\x4f\x79\x38\xfd\xee\xfb\x7f\xfd\xf2\x69\xf6\xbc\xb8\xbf\x9d\x3d\x2f\x6e\xee\x66\x3f\xd4\xf7\x30\xfa\x33\x17\x30\xf5\x19\x18\xb4\x3e\xa3\x63\x66\xb0\xb1\x0e\x06\xeb\x3b\x50\xd8\xf6\x96\x2d\x64\xed\x1d\x5b\xc8\xea\x3b\x0a\x43\xba\xe2\x31\xfd\xee\xfb\xe5\xdd\xec\xee\xf9\xf3\xe3\xfc\xf9\xf6\x61\xfe\xeb\xec\xa1\x05\x09\x64\x5a\x77\x4d\x51\x8c\x9a\xf8\x47\xad\xf8\xf8\xd9\xa8\xb8\x19\x22\x6b\x0e\x22\x2c\x59\x45\xfd\xa7\xd5\x7e\x9a\x8b\xb9\xd0\x92\xe2\x26\xc8\xd3\x6f\xec\xac\xe7\xea\x49\x07\xcd\xd3\xec\xe1\x6e\xbe\xb8\x79\x9a\xdf\x2f\x9e\xbf\xde\x7f\x79\x5e\xde\x3c\xfd\xb3\x8d\x29\x20\x3e\xc6\xda\x47\x30\x71\xf9\x9d\x86\x27\x54\xd4\x39\xaa\xcf\x16\xdd\xa3\xba\x39\xb5\xdb\xc3\x63\x1a\x41\x40\xf2\x85\x76\xe0\x06\x8c\x4a\x6a\xb2\xe6\xde\x65\x22\xc4\x52\x09\xce\xb2\x80\xcc\xd7\x0b\x85\x4b\x03\x16\xe4\xde\xa1\xb5\xf2\xdf\x8a\x7f\x0b\x2c\x31\x1c\xb3\xcf\x6e\x2e\x7a\xc1\x66\xe5\xa1\xe1\xbd\x14\xd9\x83\x52\x79\xb4\xd8\xcc\x22\xc4\x01\x41\x93\xec\xed\x57\x33\xc6\x1d\x58\xeb\xc0\xe4\xd6\x1e\xb4\x54\x31\x83\xdc\x39\x6a\x66\xeb\x81\x11\xbb\x27\xa5\x7c\x1e\x77\x35\x34\xc5\x15\xaa\x6c\xca\x43\xf8\x90\xa4\xe5\x1d\x39\x2d\x92\x88\x4b\xcf\x25\x27\xa0\x17\x72\x73\x40\x16\x63\xdd\x91\xc5\x58\xd7\x24\x3c\x42\x4d\xd4\x40\xdd\x2e\x20\xb9\xb7\x68\x18\xba\xa2\x3a\x3d\xa5\x44\x78\x6b\xa0\x98\x18\xf0\x22\x8a\x60\xa7\x4f\x4a\x2b\xa1\xa2\x6c\xda\x30\xb4\xdb\x67\x5d\x69\x40\x0f\x0f\x6e\x40\x1e\x83\x4a\x70\xfa\x8f\x4e\x30\xfd\x96\xd0\xcc\x8d\x08\xdb\x9f\x6c\x85\xa8\x46\xa4\x83\xf4\x47\x57\x8d\x2e\x8a\xaa\x3e\x62\xfe\x6a\xe1\x75\x52\xa4\x9c\xe1\xef\x83\x78\x4c\x22\x6f\xec\x42\x49\x87\xa6\x85\x21\x5f\xfa\xc5\x82\x29\xf8\xf4\xee\xa4\xfa\x84\x31\xc4\x51\xf6\xf7\xa8\x75\x2c\x88\x35\x66\xb7\xdc\x04\x44\x26\x42\xb4\x13\xf5\x00\x7a\x6f\x30\x0b\x2c\x30\x03\x0d\x13\x17\x4f\x5a\xd8\x76\xc2\xc5\xaa\xed\x81\xf4\x9f\xff\x8e\xfa\x53\x60\x98\xb8\xde\x52\x88\x95\x7c\x43\xda\xea\xaa\xf7\x30\x61\x3d\x93\xa2\x96\x07\x5e\x02\xed\x8f\x30\xcf\x9a\xda\x77\xe4\x9c\xaf\x4e\x9c\x42\x70\x19\x70\x47\x25\x8d\xc0\x4c\x45\x1a\x1f\x63\x55\x8d\x7b\x0f\xf0\xa9\x4e\x85\x3c\x89\xf4\xec\x72\xae\x45\xb5\x4a\xa9\xe5\xfd\xed\xf3\x7c\xf9\x43\xce\xb8\x26\x27\x30\xb4\x9e\x64\xee\xb2\xb5\x77\x61\x5a\xd6\x0d\x84\xae\xd8\x4d\xfd\x94\x1a\x5f\xf0\x95\x7f\x8c\x71\x9d\xc7\xcf\x3c\xe2\xb9\xdb\x2e\xc1\x30\x90\x48\x23\x98\x4e\x6a\xd7\xfc\x13\xb0\xb7\x9a\x83\xdf\x01\x0a\x52\x4c\xec\x58\xab\xb0\xa6\xed\xd5\xc8\xdf\x2b\x12\xd2\xff\x13\x1e\xa9\x0d\x4f\xb9\x80\x08\xc2\x56\x63\x6d\xb4\xd6\xeb\xb7\xa7\x94\x55\xf2\x6c\x93\x15\x08\x40\xbf\x68\xb3\x36\xff\xdd\x48\x40\xc8\xbf\x44\x6d\xd2\x89\x42\xde\x28\x4d\xa3\x5c\x69\x40\x3e\xf1\x90\x1b\xc8\xab\x31\x15\x1d\xe2\x91\xef\xcf\xcb\xc8\x00\xcb\xec\x00\x29\xde\xfd\x5d\xaa\xd3\x89\x0f\x68\xeb\x67\xd2\x2e\x97\x86\x58\x74\x08\x69\x47\x26\x84\x74\x40\x8f\xcd\xba\x5a\x6c\x36\x88\xac\x87\xb9\x1d\xa5\xe8\x95\xf1\x06\x72\xe8\x02\x13\xba\xc3\xf2\x3a\x7b\x1e\xd5\x2f\x3d\x58\xf5\x90\x22\x2e\x75\xa3\x54\xb7\xa3\xed\x40\x31\xee\xed\x7b\xbd\xa3\x44\x73\x57\x6f\x75\x7e\xad\x11\xb4\x67\x6a\xc8\xe3\xa6\x48\xfe\xdd\xad\x4d\x90\x4e\xc6\x3f\x5e\x38\x3f\xb4\xcf\xfa\xf0\xd9\xe1\x68\x04\xd6\x9d\x7d\x68\x42\x2d\x22\x61\x7f\x46\xfd\xab\xb8\x3d\x98\xf2\x65\x5d\xe1\x8a\x43\xa3\xc2\x46\xd9\x42\x73\xbd\xa6\x0e\x57\xb4\xc1\xdc\x28\xbe\x8c\xbe\xcd\xb3\x41\x99\xec\xde\x7c\xae\xde\xed\x0e\x5a\xe1\x12\x20\xcf\x15\xb7\xf2\xcf\xd6\x7f\xc0\x8e\x17\x99\x63\xa8\xc0\x9f\x86\xa6\xb7\xac\x9f\x87\xa5\x59\xe3\x4f\x53\xdb\xaa\xec\xde\xc1\xda\x7d\xe2\xec\x97\x0b\xb6\xe7\xbe\x13\x2e\xf1\x0a\xf1\x74\xa0\xc2\x0e\x28\x6f\x76\xa0\xd3\xb4\x34\x3b\xd4\xe0\xf1\xcd\x76\x75\x62\x52\x94\xed\xac\x3d\x07\x37\x5f\xba\xfb\xe9\x64\x05\x48\x77\x2f\x63\x1f\xe7\xb7\x05\x59\x7a\x9b\xb1\xb8\xe6\x94\x6a\xdc\x2b\x5e\xef\x3f\x94\xaf\x0e\x02\x92\x7f\x81\x9e\xbf\x11\x0b\xe7\x72\xad\xee\x65\x5e\x15\x77\xf5\xb2\x28\x39\x5f\xf9\x1a\x58\xc6\x04\xdc\xa9\xb0\xfa\x6b\x81\xe5\xee\x8f\x5d\xf2\x5f\x67\x7a\x03\x31\x18\x2a\x46\xff\x0b\x00\x00\xff\xff\xf6\xb2\x6e\xe3\xef\x27\x00\x00") +var _deployKubernetes119LvmPmemCsiYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5b\x6f\xe3\xb8\x15\x7e\xf7\xaf\x60\xa7\xfb\xb0\x0b\x54\x56\xbc\x45\x81\x85\x00\x3f\x64\x27\xde\xa9\xd1\x49\x62\x24\xd9\x7d\x0d\x18\xea\x58\x66\x4d\x91\x5c\xf2\x48\x13\x6d\xd1\xff\x5e\x90\x92\x6c\xdd\xac\xd8\x6e\x2e\x68\x3d\x0f\xa3\x48\x24\xcf\x77\xee\xdf\xb1\xfc\x67\xf2\x05\x24\x18\x8a\x10\x93\x6f\x1c\x37\xe4\x53\x4a\xb7\x40\xb6\x99\x45\x95\xf2\x3f\xe0\xd3\x5f\x48\xac\x88\x54\x48\x20\xe6\xf8\xa7\xc9\x84\x6a\xfe\x1b\x18\xcb\x95\x8c\x48\x3e\x9b\x6c\xb9\x8c\x23\x72\x0f\x26\xe7\x0c\x2e\x19\x53\x99\xc4\x49\x0a\x48\x63\x8a\x34\x9a\x10\x22\xe8\x13\x08\xeb\xae\x08\xd1\x29\xa4\x01\xb3\x7c\xca\x25\x82\x98\x32\x95\x86\x31\x68\xa1\x8a\x14\x24\x46\x44\xe4\x69\xa0\x8d\x8a\x33\x86\x5c\xc9\x09\x21\x92\xa6\x10\xed\x76\x05\x4c\x49\x34\x4a\x08\x30\xd5\x33\xab\x29\x83\x88\xc4\xb0\xa6\x99\xc0\x49\x10\x04\x2d\x78\xe6\x89\xb2\x29\xcd\x70\xa3\x0c\xff\x83\xba\x33\xa7\xdb\x9f\xec\x94\xab\x70\x07\xfc\x4e\x09\x78\x2b\xb8\xf0\x8c\x60\x24\x15\x6e\x51\xce\x1d\x24\x30\x01\x5b\x27\xc3\xe0\x4d\x26\xc0\x46\x93\x80\x50\xcd\xbf\x18\x95\x69\x0f\x22\x20\x9f\x3e\x4d\x08\x31\x60\x55\x66\x18\x54\xf7\x40\xc6\x5a\x71\x89\x76\x42\x48\x0e\xe6\xa9\xba\x9d\x00\xfa\xff\xbf\x51\x64\x1b\x7f\x25\xb8\x2d\x6f\xc5\x20\x00\xc1\x5f\x66\x3a\xa6\xd5\x25\x33\xe0\x2e\x7b\x32\x99\x52\x26\xe6\xb2\x69\xb2\x3e\x08\x01\xd4\xc2\x5b\x21\xb0\xa8\x0c\x4d\xe0\xa0\x70\x66\x79\xb5\x84\x51\x4d\x19\x47\x7e\x08\xca\x0e\xc0\x1e\x53\x25\xb4\x03\x45\xef\x9e\x57\x50\x8f\x72\x85\x56\xf1\x80\xe0\xde\x56\xaa\xb5\xed\x6f\x36\xa0\x05\x67\xd4\xc2\x90\x27\xcf\x88\xe6\xcf\x22\xb3\x08\xe6\xdd\x83\xda\x64\x52\x82\x39\x29\x84\xb5\x53\xcb\x22\x48\xcc\x95\xc8\xd2\xf3\xbc\x77\x92\xa3\x3a\x02\x99\xa0\x3c\x3d\x5e\x6a\x15\x28\x27\x07\x6a\x1d\xa5\x82\xda\x83\xd9\xd2\x11\x77\x5c\x09\xc8\xa1\x97\xff\x27\x86\x7a\x5f\x17\x49\xb5\xdd\x28\x9c\xbe\xa4\x54\xe5\xb2\x6a\xf9\x98\x56\xaf\x25\xc3\x95\xfe\xbe\xc2\x2f\x88\x7a\xb9\x86\x48\x15\xbf\xa6\x53\xfe\x8b\xe3\x8e\x33\x07\x45\xa4\x6c\x93\xbe\x60\x89\x86\x98\xf3\x3a\xe2\xcf\x5c\xc6\x5c\x26\x6f\xd6\xc7\x2d\x6f\x97\x0f\x25\x60\xa4\x31\x2a\x01\x77\xb0\x76\x92\x6b\x9b\x8d\x28\x32\x21\xa4\xd1\xd9\x8f\xef\xc9\x36\x7b\xfa\x27\x30\xf4\xe5\x6b\x90\xd3\xbc\x07\x1b\x69\xd4\xef\x0f\x70\xc1\xb9\x96\x6e\x76\x9d\x93\xfa\xc5\xdb\xdb\xbc\x4b\x50\x3f\x8a\x99\x5a\x0d\xcc\x09\xd2\xca\xe0\x8e\x38\x18\x8c\xc8\xec\xe2\xe2\xe2\xc2\x23\x40\x6a\x12\xc0\x55\xeb\xae\x05\x01\x0c\x95\x29\x31\x52\xad\x0f\x89\x3c\x55\x81\x8f\xb2\x53\x0a\x68\x38\xb3\xa7\x1b\x69\x36\x68\xa4\xd9\x9b\x1a\x89\x10\x2c\x34\x44\xe4\x46\xc5\xe0\x44\xf6\xac\xe6\x48\xdd\x3e\x79\xef\x91\x22\xac\x33\x71\x0f\x1f\x36\x00\xd5\x16\xac\x79\x65\x44\x66\x3d\x03\xa5\xae\x31\x7c\x6d\x60\x7a\xc1\x66\xa7\x83\xb6\x65\x0c\xdd\x8c\x60\x47\x48\xb5\xa0\x08\x15\xa4\x86\xb5\x3c\x1e\x29\x15\xfa\x52\xb3\x83\x38\x88\xc2\x32\x43\x9d\x83\xdc\xd1\x94\x4b\x30\xb6\x5a\x2d\x5a\xea\xbd\xa8\xe0\xe9\x2a\x1e\xdc\xf5\x0d\x9e\x36\x4a\x6d\x23\xc2\x13\xa9\x0c\xf8\xa5\xb5\x57\x3c\x92\xf5\x9a\x4b\x8e\xc5\x1e\x9b\xe3\x0c\x97\xbd\xbb\xce\x87\xbf\x67\xdc\x40\x7c\x95\x19\x2e\x93\x7b\xb6\x81\x38\x13\x5c\x26\x4b\x7f\x70\x75\x7b\xf1\x0c\x2c\x73\x98\x9a\x3b\xcb\x33\xef\x2b\xaf\x3f\x80\x49\x6d\xfb\x71\x50\x06\xc1\xe2\x59\x1b\xb0\xb6\x6d\xe6\x7a\xc5\x16\x8a\x68\x48\xc1\x01\xe3\xd5\x1f\xa5\xc1\x50\x17\x67\xe4\x46\xe1\x52\xf6\x9e\xe7\x54\x64\xd0\x13\xe5\xb9\x94\x54\x9f\x86\x6e\xaf\xa9\xb0\x50\x3f\xd9\x7b\xb9\x3e\xc2\x0d\xaa\x69\x4a\x65\xbc\x3f\x33\x20\x61\x66\x4d\x28\x14\xa3\x22\x7c\xe2\x32\xdc\x79\x3d\x36\x3c\x6f\x80\x0e\x48\x90\xcf\xff\xda\xfc\x33\x55\x31\xcc\x07\xd4\x0b\x48\x50\xcf\xdb\xf3\x4c\xf2\xe7\x28\x0c\x43\x66\x79\xd8\x0e\xa5\xa9\x55\x6c\xdb\xdc\x63\x20\xe1\x16\x4d\xb1\xa8\xf7\x22\xd3\x51\x18\x5e\x4c\xfd\xbf\x68\x5f\xf8\xab\xf5\xce\x67\x3c\x9e\x7f\xf7\xfd\x3f\x7e\xfd\x79\xf1\x78\x73\x7b\xb5\x78\xbc\xb9\xbc\x5e\xfc\xd0\x5c\xc3\xe8\x2f\x5c\xc0\x3c\x64\x60\xd0\x86\x8c\x4e\x99\xc1\xd6\x73\x30\xd8\x5c\x81\xc2\x76\x97\x6c\xa1\xe8\xae\xd8\x42\xd1\x5c\x51\x1a\xca\x55\x97\xf9\x77\xdf\xaf\xae\x17\xd7\x8f\x9f\xef\x97\x8f\x57\x77\xcb\xdf\x16\x77\x7d\x48\x55\x45\xff\xea\x47\xab\x79\xb4\x2f\xd5\xee\x03\x32\x6f\xba\xa6\xac\x66\x6d\xfd\x26\x9d\xf8\xf8\xc5\xa8\xb4\x1d\x22\x6b\x0e\x22\xae\x68\x49\xf3\xd3\xe9\x5f\xed\x87\x7e\xd3\x8a\xe2\x26\xf2\xe9\x37\x75\xd6\x75\x05\xa9\x87\xe6\x61\x71\x77\xbd\xbc\xb9\x7c\x58\xde\xde\x3c\x7e\xbd\xfd\xf2\xb8\xba\x7c\xf8\x7b\x17\x53\x44\x42\x4c\x75\x88\x60\xd2\xea\x4b\x91\x40\xa8\xa4\x77\xd4\x90\xad\xfa\x47\xf5\x73\x6a\xb7\x86\xa7\x34\x81\x88\xf8\x07\xdd\xc0\x8d\x18\x95\xd4\x14\xed\xb5\xab\x4c\x88\x95\x12\x9c\x15\x11\x59\xae\x6f\x14\xae\x0c\x58\x90\x7b\x87\x37\xfa\x47\x27\xfe\x77\x4d\xb6\xd6\x61\x97\x5e\xab\x4e\xcf\x6d\x9e\xb4\xef\xdf\xe5\xc7\x02\xcb\x0c\xc7\xe2\xb3\x1b\xcf\x9e\xb1\x5d\xbf\x68\x7c\x2b\x45\x71\xa7\x94\x8f\x49\x5b\x58\x84\x34\x22\x68\xb2\xbd\x17\x1a\x26\xbd\x06\x6b\x9d\x4a\xde\x67\xa3\xf6\x2e\x47\xa1\x6b\xc7\x10\x5b\x1a\xa4\xee\x4e\xb5\xdf\x47\x77\x0f\x7e\x9d\x93\x3e\x51\x0e\xed\xb4\xbc\xb7\x4f\x8b\x2c\xe1\x32\x70\x29\x0e\x18\xc4\xdc\x1c\xd8\x8b\xa9\xee\xed\xc5\x54\x37\x76\x04\x84\x9a\xa4\x85\xba\x5b\x86\xbc\xcf\x69\x1c\xbb\xd2\x3c\x3f\xa6\xd0\x04\x6b\xa0\x98\x19\x08\x12\x8a\x60\xe7\x0f\x4a\x2b\xa1\x92\x62\xde\x32\xb4\x5b\x67\x9d\xef\x30\xc0\x83\x0b\x90\xa7\xa0\x32\x9c\xff\x2d\x6d\xdd\xae\x9c\xbe\x03\xe5\x93\x7c\xd6\x8d\xda\xdf\x33\x5a\xb8\x61\x66\xfb\x93\xad\x41\x37\x28\x7f\x94\xff\xe8\xca\xde\x59\xe1\x3b\x34\x42\x9c\x14\xc7\xb3\x37\x8f\xe3\xa3\x42\xf2\x84\xc0\x3a\x88\xc7\x64\xf2\xd2\xde\x28\xe9\xd0\x74\x30\xf8\x47\xbf\x5a\x30\xe5\xfc\xb0\x3b\xa9\x39\x51\x8d\x71\xb2\xbd\x1e\x8d\x06\x0b\xa9\xc6\xe2\x8a\x9b\x88\xc8\x4c\x88\x6e\x5d\x39\x80\x3e\x18\x4d\x37\x0b\xcc\x40\xcb\xc4\xe5\x9d\x0e\xb6\xdd\xe6\xf2\xa9\x1d\x80\xf4\xaf\x7f\x77\x00\xd5\xb9\x36\x4e\xd4\xaf\x28\xa4\x4a\xbe\x21\x4d\x77\xcd\x66\x9c\xa0\x9f\x48\xc9\xab\x03\xcf\x81\xf6\xa1\x4c\xbb\x81\xfb\x1d\x39\xf6\xab\x13\xc5\x18\x5c\x0a\x5d\x53\x49\x13\x30\x73\x91\xa7\x2f\xb1\xc8\x96\xde\x23\xfc\xb1\x57\xcb\x8f\x22\x81\xbb\xa4\xed\x50\xcb\x6a\xd7\xea\xf6\xea\x71\xb9\xfa\xc1\x33\xcc\xd9\x11\x8c\x74\xa0\x1a\xf4\xd9\xe9\xbb\x30\x4f\xeb\x26\x68\x57\x2d\xe7\x61\x4e\x4d\x28\xf8\x53\x78\x04\x03\x3d\x8d\xaf\x3a\x6d\x57\x60\x18\x48\xa4\x09\xcc\x67\x6d\x35\xff\xd7\xd9\x6c\x23\x00\xde\x01\x0a\x52\xcc\xec\x54\xab\xb8\x21\xed\xd5\xc8\xf0\x2b\x12\xf4\xff\x2b\x5e\xad\x0d\xcf\xb9\x80\x04\xe2\x4e\xff\x6f\x31\x80\x8b\xb7\xa7\xd8\x75\x8a\x6e\xb3\x27\x10\x80\x61\xc9\x06\xac\xff\xdb\x48\x40\xf0\xdf\x6d\xb7\x59\x4f\xb9\xdf\x28\x4d\x13\x2f\x34\x22\x3f\xf3\x98\x1b\xf0\x35\x9f\x8a\xbe\x5d\xdc\x7a\x5f\xac\x46\x58\x77\x0f\x48\xf9\x4a\xf6\x5c\x99\x6e\xfb\x88\xb4\xe1\xc9\xc2\x65\xe4\xd8\x54\x11\x43\xde\xdb\x13\x43\x3e\x22\xc7\x16\x7d\x29\xb6\x18\x45\x36\x40\x30\x5f\x1c\x59\x6a\xe3\x8d\x64\xe2\x19\x26\x74\x87\xf9\x6a\x7e\xda\xe8\x53\x79\xb0\xee\x54\x65\x5c\xea\x56\x43\xe8\x46\xdb\x81\x92\x3f\xd8\x5d\x07\x47\xab\xf6\xaa\xc1\x1a\xff\x5a\x83\xfd\xc0\x88\xe4\xe3\xa6\x2c\x21\x3b\xad\x4d\x94\xcf\xa6\x3f\x9e\x39\x2c\x75\xcf\xfa\xf0\x11\xe7\xc5\x08\x6c\x3a\xfb\xd0\xc4\x5e\x46\xc2\xfe\x8c\xe6\x17\x9c\x7b\x30\xd5\x3b\xd4\xd2\x15\x87\x26\x9a\x8d\xb2\xa5\xe4\x66\x4d\x1d\xaf\x68\xa3\xb9\x51\xbe\x23\xb8\xf2\xd9\xa0\x4c\x71\x6b\x3e\xd7\xaf\xdc\x47\xad\x70\x0e\x90\xc7\x9a\xc1\x85\x27\xcb\x3f\x60\xc7\xb3\xcc\x31\x56\xe0\x8f\x43\x33\x58\xd6\x4f\xc3\xd2\xae\xf1\xc7\x89\xed\x54\xf6\xe0\x60\xed\x3e\x72\x44\xf5\x1b\xbb\xe3\xe9\x11\x4a\xbc\x42\x3c\x1d\xa8\xb0\x23\xc2\xdb\x1d\xe8\x38\x29\xed\x0e\x35\x7a\x7c\xbb\x5d\x1d\x99\x14\x55\x3b\xeb\x8e\xeb\xed\xdf\x42\x84\xf9\xec\x09\x90\xee\xde\x91\xdf\x2f\xaf\x4a\xca\xf5\x36\xd3\x7b\xc3\x29\xf5\x50\x59\xfe\xea\xe2\xae\x7a\x21\x13\x11\xff\x5a\xc2\xbf\xa8\x8c\x97\x72\xad\x6e\xa5\xaf\x8a\xbb\x7a\x59\x96\x9c\xaf\x7c\x0d\xac\x60\x02\xae\x55\x5c\xff\x88\x63\xb5\xfb\x0d\x92\xff\x73\xa1\x37\x90\x82\xa1\x62\xf2\x9f\x00\x00\x00\xff\xff\xb2\xb2\xba\xec\x86\x29\x00\x00") func deployKubernetes119LvmPmemCsiYamlBytes() ([]byte, error) { return bindataRead( @@ -239,7 +239,7 @@ func deployKubernetes119LvmPmemCsiYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/kubernetes-1.19/lvm/pmem-csi.yaml", size: 10223, mode: os.FileMode(420), modTime: time.Unix(1599471429, 0)} + info := bindataFileInfo{name: "deploy/kubernetes-1.19/lvm/pmem-csi.yaml", size: 10630, mode: os.FileMode(420), modTime: time.Unix(1599822742, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/deploy/kubernetes-1.17/direct/pmem-csi.yaml b/deploy/kubernetes-1.17/direct/pmem-csi.yaml index d136738ff1..c68d8d7d34 100644 --- a/deploy/kubernetes-1.17/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.17/direct/pmem-csi.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: direct-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.17/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.17/direct/testing/pmem-csi.yaml index d33cdd5b5d..77accdfd55 100644 --- a/deploy/kubernetes-1.17/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.17/direct/testing/pmem-csi.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: direct-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.17/lvm/pmem-csi.yaml b/deploy/kubernetes-1.17/lvm/pmem-csi.yaml index 52dfcbabae..04f96a8cfb 100644 --- a/deploy/kubernetes-1.17/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.17/lvm/pmem-csi.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: lvm-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.17/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.17/lvm/testing/pmem-csi.yaml index 2c89e4f9a7..da450c8f68 100644 --- a/deploy/kubernetes-1.17/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.17/lvm/testing/pmem-csi.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: lvm-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.17/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.17/pmem-csi-direct-testing.yaml index d33cdd5b5d..77accdfd55 100644 --- a/deploy/kubernetes-1.17/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.17/pmem-csi-direct-testing.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: direct-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.17/pmem-csi-direct.yaml b/deploy/kubernetes-1.17/pmem-csi-direct.yaml index d136738ff1..c68d8d7d34 100644 --- a/deploy/kubernetes-1.17/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.17/pmem-csi-direct.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: direct-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.17/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.17/pmem-csi-lvm-testing.yaml index 2c89e4f9a7..da450c8f68 100644 --- a/deploy/kubernetes-1.17/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.17/pmem-csi-lvm-testing.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: lvm-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.17/pmem-csi-lvm.yaml b/deploy/kubernetes-1.17/pmem-csi-lvm.yaml index 52dfcbabae..04f96a8cfb 100644 --- a/deploy/kubernetes-1.17/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.17/pmem-csi-lvm.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: lvm-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.18/direct/pmem-csi.yaml b/deploy/kubernetes-1.18/direct/pmem-csi.yaml index d136738ff1..c68d8d7d34 100644 --- a/deploy/kubernetes-1.18/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.18/direct/pmem-csi.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: direct-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.18/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.18/direct/testing/pmem-csi.yaml index d33cdd5b5d..77accdfd55 100644 --- a/deploy/kubernetes-1.18/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.18/direct/testing/pmem-csi.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: direct-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.18/lvm/pmem-csi.yaml b/deploy/kubernetes-1.18/lvm/pmem-csi.yaml index 52dfcbabae..04f96a8cfb 100644 --- a/deploy/kubernetes-1.18/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.18/lvm/pmem-csi.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: lvm-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.18/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.18/lvm/testing/pmem-csi.yaml index 2c89e4f9a7..da450c8f68 100644 --- a/deploy/kubernetes-1.18/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.18/lvm/testing/pmem-csi.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: lvm-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.18/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.18/pmem-csi-direct-testing.yaml index d33cdd5b5d..77accdfd55 100644 --- a/deploy/kubernetes-1.18/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.18/pmem-csi-direct-testing.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: direct-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.18/pmem-csi-direct.yaml b/deploy/kubernetes-1.18/pmem-csi-direct.yaml index d136738ff1..c68d8d7d34 100644 --- a/deploy/kubernetes-1.18/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.18/pmem-csi-direct.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: direct-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.18/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.18/pmem-csi-lvm-testing.yaml index 2c89e4f9a7..da450c8f68 100644 --- a/deploy/kubernetes-1.18/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.18/pmem-csi-lvm-testing.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: lvm-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.18/pmem-csi-lvm.yaml b/deploy/kubernetes-1.18/pmem-csi-lvm.yaml index 52dfcbabae..04f96a8cfb 100644 --- a/deploy/kubernetes-1.18/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.18/pmem-csi-lvm.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: lvm-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml b/deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml index 41c40391fc..dfd56281e8 100644 --- a/deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.19-alpha/direct/pmem-csi.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -282,6 +287,7 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 env: - name: POD_NAMESPACE valueFrom: @@ -294,6 +300,9 @@ spec: image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -326,6 +335,8 @@ spec: pmem-csi.intel.com/deployment: direct-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-production @@ -347,6 +358,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -365,6 +377,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19-alpha/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.19-alpha/direct/testing/pmem-csi.yaml index a720f955a5..3f0a9d439d 100644 --- a/deploy/kubernetes-1.19-alpha/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19-alpha/direct/testing/pmem-csi.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -302,6 +307,7 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 env: - name: POD_NAMESPACE @@ -315,6 +321,9 @@ spec: image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -374,6 +383,8 @@ spec: pmem-csi.intel.com/deployment: direct-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing @@ -395,6 +406,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -416,6 +428,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml b/deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml index 227798bad1..1e6ef3fd88 100644 --- a/deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.19-alpha/lvm/pmem-csi.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -282,6 +287,7 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 env: - name: POD_NAMESPACE valueFrom: @@ -294,6 +300,9 @@ spec: image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -326,6 +335,8 @@ spec: pmem-csi.intel.com/deployment: lvm-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production @@ -347,6 +358,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -365,6 +377,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19-alpha/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.19-alpha/lvm/testing/pmem-csi.yaml index 95309651bb..a54cab22e3 100644 --- a/deploy/kubernetes-1.19-alpha/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19-alpha/lvm/testing/pmem-csi.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -302,6 +307,7 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 env: - name: POD_NAMESPACE @@ -315,6 +321,9 @@ spec: image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -374,6 +383,8 @@ spec: pmem-csi.intel.com/deployment: lvm-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing @@ -395,6 +406,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -416,6 +428,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19-alpha/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.19-alpha/pmem-csi-direct-testing.yaml index a720f955a5..3f0a9d439d 100644 --- a/deploy/kubernetes-1.19-alpha/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.19-alpha/pmem-csi-direct-testing.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -302,6 +307,7 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 env: - name: POD_NAMESPACE @@ -315,6 +321,9 @@ spec: image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -374,6 +383,8 @@ spec: pmem-csi.intel.com/deployment: direct-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing @@ -395,6 +406,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -416,6 +428,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19-alpha/pmem-csi-direct.yaml b/deploy/kubernetes-1.19-alpha/pmem-csi-direct.yaml index 41c40391fc..dfd56281e8 100644 --- a/deploy/kubernetes-1.19-alpha/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.19-alpha/pmem-csi-direct.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -282,6 +287,7 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 env: - name: POD_NAMESPACE valueFrom: @@ -294,6 +300,9 @@ spec: image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -326,6 +335,8 @@ spec: pmem-csi.intel.com/deployment: direct-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-production @@ -347,6 +358,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -365,6 +377,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19-alpha/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.19-alpha/pmem-csi-lvm-testing.yaml index 95309651bb..a54cab22e3 100644 --- a/deploy/kubernetes-1.19-alpha/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.19-alpha/pmem-csi-lvm-testing.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -302,6 +307,7 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 env: - name: POD_NAMESPACE @@ -315,6 +321,9 @@ spec: image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -374,6 +383,8 @@ spec: pmem-csi.intel.com/deployment: lvm-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing @@ -395,6 +406,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -416,6 +428,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19-alpha/pmem-csi-lvm.yaml b/deploy/kubernetes-1.19-alpha/pmem-csi-lvm.yaml index 227798bad1..1e6ef3fd88 100644 --- a/deploy/kubernetes-1.19-alpha/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.19-alpha/pmem-csi-lvm.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -282,6 +287,7 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 env: - name: POD_NAMESPACE valueFrom: @@ -294,6 +300,9 @@ spec: image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -326,6 +335,8 @@ spec: pmem-csi.intel.com/deployment: lvm-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production @@ -347,6 +358,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -365,6 +377,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19/direct/pmem-csi.yaml b/deploy/kubernetes-1.19/direct/pmem-csi.yaml index d136738ff1..c68d8d7d34 100644 --- a/deploy/kubernetes-1.19/direct/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/direct/pmem-csi.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: direct-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml b/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml index d33cdd5b5d..77accdfd55 100644 --- a/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/direct/testing/pmem-csi.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: direct-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19/lvm/pmem-csi.yaml b/deploy/kubernetes-1.19/lvm/pmem-csi.yaml index 52dfcbabae..04f96a8cfb 100644 --- a/deploy/kubernetes-1.19/lvm/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/lvm/pmem-csi.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: lvm-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml b/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml index 2c89e4f9a7..da450c8f68 100644 --- a/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml +++ b/deploy/kubernetes-1.19/lvm/testing/pmem-csi.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: lvm-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml b/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml index d33cdd5b5d..77accdfd55 100644 --- a/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-direct-testing.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: direct-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19/pmem-csi-direct.yaml b/deploy/kubernetes-1.19/pmem-csi-direct.yaml index d136738ff1..c68d8d7d34 100644 --- a/deploy/kubernetes-1.19/pmem-csi-direct.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-direct.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: direct-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: direct-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: direct-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml b/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml index 2c89e4f9a7..da450c8f68 100644 --- a/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-lvm-testing.yaml @@ -239,6 +239,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-testing @@ -261,12 +263,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-controller-*.out @@ -283,6 +285,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -301,10 +306,14 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 - -v=5 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -364,6 +373,8 @@ spec: pmem-csi.intel.com/deployment: lvm-testing template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-testing @@ -385,6 +396,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 - -v=5 - -testEndpoint - -coverprofile=/var/lib/pmem-csi-coverage/pmem-csi-driver-node-*.out @@ -406,6 +418,9 @@ spec: image: intel/pmem-csi-driver-test:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kubernetes-1.19/pmem-csi-lvm.yaml b/deploy/kubernetes-1.19/pmem-csi-lvm.yaml index 52dfcbabae..04f96a8cfb 100644 --- a/deploy/kubernetes-1.19/pmem-csi-lvm.yaml +++ b/deploy/kubernetes-1.19/pmem-csi-lvm.yaml @@ -224,6 +224,8 @@ spec: serviceName: pmem-csi-controller template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-controller pmem-csi.intel.com/deployment: lvm-production @@ -246,12 +248,12 @@ spec: - -mode=controller - -endpoint=unix:///csi/csi-controller.sock - -registryEndpoint=tcp://0.0.0.0:10000 - - -metricsListen=:10010 - -nodeid=$(KUBE_NODE_NAME) - -caFile=/certs/ca.crt - -certFile=/certs/tls.crt - -keyFile=/certs/tls.key - -drivername=$(PMEM_CSI_DRIVER_NAME) + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -265,6 +267,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: readOnlyRootFilesystem: true terminationMessagePath: /tmp/termination-log @@ -281,9 +286,13 @@ spec: - --feature-gates=Topology=true - --strict-topology=true - --timeout=5m + - --metrics-address=:10011 image: quay.io/k8scsi/csi-provisioner:v2.0.0 imagePullPolicy: IfNotPresent name: external-provisioner + ports: + - containerPort: 10011 + name: metrics securityContext: readOnlyRootFilesystem: true volumeMounts: @@ -316,6 +325,8 @@ spec: pmem-csi.intel.com/deployment: lvm-production template: metadata: + annotations: + pmem-csi.intel.com/scrape: containers labels: app: pmem-csi-node pmem-csi.intel.com/deployment: lvm-production @@ -337,6 +348,7 @@ spec: - -statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME) - -drivername=$(PMEM_CSI_DRIVER_NAME) - -pmemPercentage=100 + - -metricsListen=:10010 env: - name: KUBE_NODE_NAME valueFrom: @@ -355,6 +367,9 @@ spec: image: intel/pmem-csi-driver:canary imagePullPolicy: IfNotPresent name: pmem-driver + ports: + - containerPort: 10010 + name: metrics securityContext: privileged: true runAsUser: 0 diff --git a/deploy/kustomize/driver/pmem-csi.yaml b/deploy/kustomize/driver/pmem-csi.yaml index 4b71ff2279..483611fe5d 100644 --- a/deploy/kustomize/driver/pmem-csi.yaml +++ b/deploy/kustomize/driver/pmem-csi.yaml @@ -84,7 +84,6 @@ spec: "-mode=controller", "-endpoint=unix:///csi/csi-controller.sock", "-registryEndpoint=tcp://0.0.0.0:10000", - "-metricsListen=:10010", "-nodeid=$(KUBE_NODE_NAME)", "-caFile=/certs/ca.crt", "-certFile=/certs/tls.crt", @@ -116,7 +115,12 @@ spec: - name: external-provisioner image: quay.io/k8scsi/csi-provisioner:v1.X.Y imagePullPolicy: IfNotPresent - args: [ "-v=3", "--csi-address=/csi/csi-controller.sock", "--feature-gates=Topology=true", "--strict-topology=true", "--timeout=5m" ] + args: + - -v=3 + - --csi-address=/csi/csi-controller.sock + - --feature-gates=Topology=true + - --strict-topology=true + - --timeout=5m securityContext: readOnlyRootFilesystem: true volumeMounts: diff --git a/deploy/kustomize/kubernetes-base-direct/kustomization.yaml b/deploy/kustomize/kubernetes-base-direct/kustomization.yaml index 626e9a1177..7e45e8976c 100644 --- a/deploy/kustomize/kubernetes-base-direct/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-direct/kustomization.yaml @@ -1,7 +1,7 @@ -# Turns generic Kubernetes 1.16 deployment into deployment for direct mode. +# Turns the base deployment with metrics into a deployment for direct mode. bases: -- ../kubernetes-base/ +- ../kubernetes-with-metrics commonLabels: pmem-csi.intel.com/deployment: direct-production diff --git a/deploy/kustomize/kubernetes-base-lvm/kustomization.yaml b/deploy/kustomize/kubernetes-base-lvm/kustomization.yaml index 7b24a0aeee..955e914373 100644 --- a/deploy/kustomize/kubernetes-base-lvm/kustomization.yaml +++ b/deploy/kustomize/kubernetes-base-lvm/kustomization.yaml @@ -1,7 +1,7 @@ -# Turns generic Kubernetes 1.16 deployment into deployment for LVM mode. +# Turns the base deployment with metrics support into a deployment for LVM mode. bases: -- ../kubernetes-base/ +- ../kubernetes-with-metrics commonLabels: pmem-csi.intel.com/deployment: lvm-production diff --git a/deploy/kustomize/kubernetes-base/README.md b/deploy/kustomize/kubernetes-no-metrics/README.md similarity index 100% rename from deploy/kustomize/kubernetes-base/README.md rename to deploy/kustomize/kubernetes-no-metrics/README.md diff --git a/deploy/kustomize/kubernetes-base/driverinfo-beta.yaml b/deploy/kustomize/kubernetes-no-metrics/driverinfo-beta.yaml similarity index 100% rename from deploy/kustomize/kubernetes-base/driverinfo-beta.yaml rename to deploy/kustomize/kubernetes-no-metrics/driverinfo-beta.yaml diff --git a/deploy/kustomize/kubernetes-base/kustomization.yaml b/deploy/kustomize/kubernetes-no-metrics/kustomization.yaml similarity index 100% rename from deploy/kustomize/kubernetes-base/kustomization.yaml rename to deploy/kustomize/kubernetes-no-metrics/kustomization.yaml diff --git a/deploy/kustomize/kubernetes-base/rbac/README.md b/deploy/kustomize/kubernetes-no-metrics/rbac/README.md similarity index 100% rename from deploy/kustomize/kubernetes-base/rbac/README.md rename to deploy/kustomize/kubernetes-no-metrics/rbac/README.md diff --git a/deploy/kustomize/kubernetes-base/rbac/kustomization.yaml b/deploy/kustomize/kubernetes-no-metrics/rbac/kustomization.yaml similarity index 100% rename from deploy/kustomize/kubernetes-base/rbac/kustomization.yaml rename to deploy/kustomize/kubernetes-no-metrics/rbac/kustomization.yaml diff --git a/deploy/kustomize/kubernetes-with-metrics/kustomization.yaml b/deploy/kustomize/kubernetes-with-metrics/kustomization.yaml new file mode 100644 index 0000000000..48f98053d8 --- /dev/null +++ b/deploy/kustomize/kubernetes-with-metrics/kustomization.yaml @@ -0,0 +1,19 @@ +# Enables metrics support in the base deployment. + +bases: +- ../kubernetes-no-metrics + +patchesJson6902: +- target: + group: apps + version: v1 + kind: DaemonSet + name: pmem-csi-node + path: ../patches/metrics-node.yaml + +- target: + group: apps + version: v1 + kind: StatefulSet + name: pmem-csi-controller + path: ../patches/metrics-controller.yaml diff --git a/deploy/kustomize/patches/metrics-controller.yaml b/deploy/kustomize/patches/metrics-controller.yaml new file mode 100644 index 0000000000..f2c680ad52 --- /dev/null +++ b/deploy/kustomize/patches/metrics-controller.yaml @@ -0,0 +1,30 @@ +# This JSON patch adds the necessary annotation, port definitions and +# arguments to the PMEM-CSI controller pod. + +# PMEM-CSI: +- op: add + path: /spec/template/metadata/annotations + value: + pmem-csi.intel.com/scrape: containers +- op: add + path: /spec/template/spec/containers/0/ports + value: + - name: metrics + containerPort: 10010 +- op: add + path: /spec/template/spec/containers/0/command/- + value: -metricsListen=:10010 + +# external-provisioner: +- op: add + path: /spec/template/metadata/annotations + value: + pmem-csi.intel.com/scrape: containers +- op: add + path: /spec/template/spec/containers/1/ports + value: + - name: metrics + containerPort: 10011 +- op: add + path: /spec/template/spec/containers/1/args/- + value: --metrics-address=:10011 diff --git a/deploy/kustomize/patches/metrics-node.yaml b/deploy/kustomize/patches/metrics-node.yaml new file mode 100644 index 0000000000..4ed47c8b7c --- /dev/null +++ b/deploy/kustomize/patches/metrics-node.yaml @@ -0,0 +1,18 @@ +# This JSON patch adds the necessary annotation, port definitions and +# arguments to the PMEM-CSI node pod. + +# PMEM-CSI: +- op: add + path: /spec/template/metadata/annotations + value: + pmem-csi.intel.com/scrape: containers +- op: add + path: /spec/template/spec/containers/0/ports + value: + - name: metrics + containerPort: 10010 +- op: add + path: /spec/template/spec/containers/0/command/- + value: -metricsListen=:10010 + +# TODO: node-driver-registrar once it has metrics support. diff --git a/deploy/prometheus.yaml b/deploy/prometheus.yaml new file mode 100644 index 0000000000..ff8345e4e1 --- /dev/null +++ b/deploy/prometheus.yaml @@ -0,0 +1,42 @@ +extraScrapeConfigs: | + # Example scrape config for PMEM-CSI containers. + # + # The relabeling allows the actual container scrape endpoints to be configured in the + # Kubernetes deployment via the "pmem-csi.intel.com/scrape" annotations and the "metrics" + # container ports. + + - job_name: 'pmem-csi-containers' + + kubernetes_sd_configs: + - role: pod + + relabel_configs: + # Only scrape containers which have the "pmem-csi.intel.com/scrape" annotation.... + - source_labels: [__meta_kubernetes_pod_annotation_pmem_csi_intel_com_scrape] + action: keep + regex: containers + # ... and a "metrics" port. + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: metrics + # Overwrite default port (if any) with the container port number. + - source_labels: [__address__, __meta_kubernetes_pod_container_port_number] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + # Add certain information from Kubernetes (namespace, pod and container names, node) + # as label to each sample. The node name is useful for daemon sets where + # pods typically have no meaningful name. + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - source_labels: [__meta_kubernetes_pod_container_name] + action: replace + target_label: kubernetes_pod_container_name + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: kubernetes_pod_node_name diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 53a828cc07..5aa934eca2 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -59,7 +59,7 @@ The normal Go style guide applies. It is enforced by `make test`, which calls `g ### Input validation -In all cases, input comes from a trusted source because network +In most cases, input comes from a trusted source because network communication is protected by mutual TLS and the `kubectl` binaries runs with the same privileges as the user invoking it. @@ -73,6 +73,8 @@ Nonetheless, input needs to be validated to catch mistakes: (https://godoc.org/google.golang.org/grpc#MaxRecvMsgSize) and refuses to send messages that are larger (https://godoc.org/google.golang.org/grpc#MaxSendMsgSize) +- webhook and metrics SDK code does input validation before + invoking PMEM-CSI ## Release management @@ -170,9 +172,16 @@ GetCapacity, GetCapabilities, GetPluginInfo, GetPluginCapabilities. Network ports are opened as configured in manifest files: - registry endpoint: typical port value 10000, used for PMEM-CSI internal communication -- controller endpoint: typical port value 10001, used for serving CSI API +- controller endpoint: typical port value 10001, used by the nodes for + providing the serving CSI API to the PMEM-CSI controller +- metrics endpoint: typical port values 10010 (PMEM-CSI) and 10011 (external-provisioner) - webhook endpoint: disabled by default, port chosen when [enabling the scheduler extensions](../README.md#enable-scheduler-extensions) +Except for the metrics and webhook endpoint, all ports are protected +via mutual TLS. The metrics endpoint and webhook are supposed to be +easily usable and expose no confidential data, therefore TLS is not +used. + ### Local sockets diff --git a/docs/install.md b/docs/install.md index 83f8fb616f..76cf0f6f56 100644 --- a/docs/install.md +++ b/docs/install.md @@ -13,6 +13,7 @@ - [Ephemeral inline volumes](#ephemeral-inline-volumes) - [Raw block volumes](#raw-block-volumes) - [Enable scheduler extensions](#enable-scheduler-extensions) + - [Metrics support](#metrics-support) - [PMEM-CSI Deployment CRD](#pmem-csi-deployment-crd) - [Filing issues and contributing](#filing-issues-and-contributing) @@ -795,7 +796,6 @@ EOF $ kubectl create --kustomize my-webhook ``` - ### Storage capacity tracking [Kubernetes @@ -826,6 +826,165 @@ INFO: deploying from /nvme/gopath/src/github.com/intel/pmem-csi/deploy/kubernete ... ``` +### Metrics support + +:bangbang: | Metric support is an alpha feature. What data is provided may change. +:---: | :--- + + +Metrics support is controlled by command line options of the PMEM-CSI +driver binary and of the CSI sidecars. Annotations and named container +ports make it possible to discover these data scraping endpoints. The +[metrics kustomize +base](/deploy/kustomize/kubernetes-with-metrics/kustomization.yaml) +adds all of that to the pre-generated deployment files. The operator +also enables the metrics support. + +Access to metrics data is not restricted (no TLS, no client +authorization) because the metrics data is not considered confidential +and access control would just make client configuration unnecessarily +complex. + +#### Metrics data + +PMEM-CSI exposes metrics data about the Go runtime, Prometheus, CSI +method calls, and PMEM-CSI: + +Name | Type | Explanation +-----|------|------------ +`build_info` | gauge | A metric with a constant '1' value labeled by version. +`csi_[sidecar\|plugin]_operations_seconds` | histogram | gRPC call duration and error code, for sidecar to driver (aka plugin) communication. +`go_*` | | [Go runtime information](https://github.com/prometheus/client_golang/blob/master/prometheus/go_collector.go) +`pmem_amount_available` | gauge | Remaining amount of PMEM on the host that can be used for new volumes. +`pmem_amount_managed` | gauge | Amount of PMEM on the host that is managed by PMEM-CSI. +`pmem_amount_max_volume_size` | gauge | The size of the largest PMEM volume that can be created. +`pmem_amount_total` | gauge | Total amount of PMEM on the host. +`pmem_csi_[controller\|node]_operations_seconds` | histogram | gRPC call duration and error code, for the PMEM-CSI internal controller to node communication. The `node` label identifies the node. +`pmem_nodes` | gauge | The number of PMEM-CSI nodes registered in the controller. +`process_*` | | [Process information](https://github.com/prometheus/client_golang/blob/master/prometheus/process_collector.go) +`promhttp_metric_handler_requests_in_flight` | gauge | Current number of scrapes being served. +`promhttp_metric_handler_requests_total` | counter | Total number of scrapes by HTTP status code. + +This list is tentative and may still change as long as metrics support +is alpha. To see all available data, query a container. Different +containers provide different data. For example, the controller +provides: + +``` ShellSession +$ kubectl port-forward pmem-csi-controller-0 10010 +Forwarding from 127.0.0.1:10010 -> 10010 +Forwarding from [::1]:10010 -> 10010 +``` + +And in another shell: + +``` ShellSession +$ curl --silent http://localhost:10010/metrics | grep '# ' +# HELP build_info A metric with a constant '1' value labeled by version. +# TYPE build_info gauge +... +# HELP csi_plugin_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total +# TYPE csi_plugin_operations_seconds histogram +... +# HELP pmem_csi_controller_operations_seconds [ALPHA] Container Storage Interface operation duration with gRPC error code status total +# TYPE pmem_csi_controller_operations_seconds histogram +... +``` + + +#### Prometheus example + +An [extension of the scrape config](/deploy/prometheus.yaml) is +necessary for [Prometheus](https://prometheus.io/). When deploying +[Prometheus via Helm](https://hub.helm.sh/charts/stable/prometheus), +that file can be added to the default configuration with the `-f` +parameter. The following example works for the [QEMU-based +cluster](autotest.md#qemu-and-kubernetes) and Helm v3.1.2. In a real +production deployment, some kind of persistent storage should be +provided. The [URL](/deploy/prometheus.yaml) can be used instead of +the file name, too. + + +``` ShellSession +$ helm install prometheus stable/prometheus \ + --set alertmanager.persistentVolume.enabled=false,server.persistentVolume.enabled=false \ + -f deploy/prometheus.yaml +NAME: prometheus +LAST DEPLOYED: Tue Aug 18 18:04:27 2020 +NAMESPACE: default +STATUS: deployed +REVISION: 1 +TEST SUITE: None +NOTES: +The Prometheus server can be accessed via port 80 on the following DNS name from within your cluster: +prometheus-server.default.svc.cluster.local + + +Get the Prometheus server URL by running these commands in the same shell: + export POD_NAME=$(kubectl get pods --namespace default -l "app=prometheus,component=server" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace default port-forward $POD_NAME 9090 +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Server pod is terminated. ##### +################################################################################# + +... +``` + +After running this `kubectl port-forward` command, it is possible to +access the [Prometheus web +interface](https://prometheus.io/docs/prometheus/latest/getting_started/#using-the-expression-browser) +and run some queries there. Here are some examples for the QEMU test +cluster with two volumes created on node `pmem-csi-pmem-govm-worker2`. + +Available PMEM as percentage: +``` +pmem_amount_available / pmem_amount_managed +``` + +Result variable | Value | Tags +----------------|-------|----- +| none | 0.7332986065893997 | instance = 10.42.0.1:10010 | +| | | job = pmem-csi-containers | +| | | kubernetes_namespace = default | +| | | kubernetes_pod_container_name = pmem-driver | +| | | kubernetes_pod_name = pmem-csi-node-dfkrw | +| | | kubernetes_pod_node_name = pmem-csi-pmem-govm-worker2 | +| | | node = pmem-csi-pmem-govm-worker2 | +| | 1 | instance = 10.36.0.1:10010 | +| | | job = pmem-csi-containers | +| | | kubernetes_namespace = default | +| | | kubernetes_pod_container_name = pmem-driver | +| | | kubernetes_pod_name = pmem-csi-node-z5vnp | +| | | kubernetes_pod_node_name pmem-csi-pmem-govm-worker3 | +| | | node = pmem-csi-pmem-govm-worker3 | +| | 1 | instance = 10.44.0.1:10010 | +| | | job = pmem-csi-containers | +| | | kubernetes_namespace = default | +| | | kubernetes_pod_container_name = pmem-driver | +| | | kubernetes_pod_name = pmem-csi-node-zzmsd | +| | | kubernetes_pod_node_name = pmem-csi-pmem-govm-worker1 | +| | | node = pmem-csi-pmem-govm-worker1 | + + +Number of `CreateVolume` calls in nodes: +``` +pmem_csi_node_operations_seconds_count{method_name="/csi.v1.Controller/CreateVolume"} +``` + +Result variable | Value | Tags +----------------|-------|------ +| `pmem_csi_node_operations_seconds_count` | 2 | driver_name = pmem-csi.intel.com | +| | | grpc_status_code = OK | +| | | instance = 10.42.0.1:10010 | +| | | job = pmem-csi-containers | +| | | kubernetes_namespace = default | +| | | kubernetes_pod_container_name = pmem-driver | +| | | kubernetes_pod_name = pmem-csi-node-dfkrw | +| | | kubernetes_pod_node_name = pmem-csi-pmem-govm-worker2 | +| | | method_name = /csi.v1.Controller/CreateVolume | +| | | node = pmem-csi-pmem-govm-worker2 | + ## PMEM-CSI Deployment CRD `Deployment` is a cluster-scoped Kubernetes resource in the @@ -853,7 +1012,7 @@ The current API for PMEM-CSI `Deployment` resources is: ### Deployment -|Field | type | Description | +|Field | Type | Description | |---|---|---| | apiVersion | string | `pmem-csi.intel.com/v1alpha1`| | kind | string | `Deployment`| @@ -862,7 +1021,7 @@ The current API for PMEM-CSI `Deployment` resources is: #### DeploymentSpec -|Field | type | Description | Default Value | +|Field | Type | Description | Default Value | |---|---|---|---| | image | string | PMEM-CSI docker image name used for the deployment | the same image as the operator1 | | provisionerImage | string | [CSI provisioner](https://kubernetes-csi.github.io/docs/external-provisioner.html) docker image name | latest [external provisioner](https://kubernetes-csi.github.io/docs/external-provisioner.html) stable release image2 | @@ -905,7 +1064,7 @@ Deployment is in it's lifecycle. The possible `phase` values and their meaning are as below: -| value | meaning | +| Value | Meaning | |---|---| | empty string | A new deployment. | | Initializing | All the direct sub-resources of the `Deployment` are created, but some indirect ones (like pods controlled by a daemon set) may still be missing. | diff --git a/go.mod b/go.mod index 3e6b11cd7e..c7f7b83654 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/golang/protobuf v1.4.2 github.com/google/uuid v1.1.1 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect - github.com/kubernetes-csi/csi-lib-utils v0.7.0 + github.com/kubernetes-csi/csi-lib-utils v0.8.0 github.com/kubernetes-csi/csi-test/v3 v3.1.0 github.com/onsi/ginkgo v1.12.1 github.com/onsi/gomega v1.10.1 @@ -22,7 +22,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 github.com/prometheus/common v0.10.0 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 go.uber.org/multierr v1.2.0 // indirect go.uber.org/zap v1.11.0 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 @@ -34,7 +34,7 @@ require ( k8s.io/api v0.19.0 k8s.io/apiextensions-apiserver v0.19.0 k8s.io/apimachinery v0.19.1-rc.0 - k8s.io/client-go v0.19.0 + k8s.io/client-go v1.19.0 k8s.io/component-base v0.19.0 k8s.io/klog v1.0.0 k8s.io/kube-scheduler v0.19.0 diff --git a/go.sum b/go.sum index 5ea1812f4f..e5ce70267e 100644 --- a/go.sum +++ b/go.sum @@ -90,7 +90,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= -github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= github.com/container-storage-interface/spec v1.2.0 h1:bD9KIVgaVKKkQ/UbVUY9kCaH/CJbhNxe0eeB4JeJV2s= github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= @@ -253,6 +252,8 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= @@ -295,7 +296,6 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= @@ -368,8 +368,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kubernetes-csi/csi-lib-utils v0.7.0 h1:t1cS7HTD7z5D7h9iAdjWuHtMxJPb9s1fIv34rxytzqs= -github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= +github.com/kubernetes-csi/csi-lib-utils v0.8.0 h1:hYbmGAEll6yODw3oy58byNyhh9VXPu8Ulyyo1a7Ejq0= +github.com/kubernetes-csi/csi-lib-utils v0.8.0/go.mod h1:yCT1Ht9rtmpJI+4dZOwuB4LsJhaqR0nW+VZ/Spa0xHE= github.com/kubernetes-csi/csi-test/v3 v3.1.0 h1:XPiXQgEhDV9y+6vHAklH273PKA/ocqQdC8R+WcIKZKw= github.com/kubernetes-csi/csi-test/v3 v3.1.0/go.mod h1:UWxYP5cDlD6iSNVKEiLFqfJnJinuhtI7MLt61rQQOfI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= @@ -432,10 +432,10 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -542,6 +542,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLYM/iQ8KXej1AwM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -696,7 +698,6 @@ golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -784,7 +785,6 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -799,6 +799,7 @@ google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/grpc-server/server.go b/pkg/grpc-server/server.go index c88b56a49e..aa5bddf8f9 100644 --- a/pkg/grpc-server/server.go +++ b/pkg/grpc-server/server.go @@ -12,6 +12,7 @@ import ( "sync" "github.com/intel/pmem-csi/pkg/pmem-grpc" + "github.com/kubernetes-csi/csi-lib-utils/metrics" "google.golang.org/grpc" "k8s.io/klog" ) @@ -32,11 +33,11 @@ func NewNonBlockingGRPCServer() *NonBlockingGRPCServer { return &NonBlockingGRPCServer{} } -func (s *NonBlockingGRPCServer) Start(endpoint string, tlsConfig *tls.Config, services ...Service) error { +func (s *NonBlockingGRPCServer) Start(endpoint string, tlsConfig *tls.Config, csiMetricsManager metrics.CSIMetricsManager, services ...Service) error { if endpoint == "" { return fmt.Errorf("endpoint cannot be empty") } - rpcServer, l, err := pmemgrpc.NewServer(endpoint, tlsConfig) + rpcServer, l, err := pmemgrpc.NewServer(endpoint, tlsConfig, csiMetricsManager) if err != nil { return nil } diff --git a/pkg/pmem-csi-driver/controllerserver-node.go b/pkg/pmem-csi-driver/controllerserver-node.go index a7ebd433f0..cb872ef871 100644 --- a/pkg/pmem-csi-driver/controllerserver-node.go +++ b/pkg/pmem-csi-driver/controllerserver-node.go @@ -391,17 +391,16 @@ func (cs *nodeControllerServer) ListVolumes(ctx context.Context, req *csi.ListVo } func (cs *nodeControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { - var capacity int64 - cap, err := cs.dm.GetCapacity() if err != nil { return nil, status.Errorf(codes.Internal, err.Error()) } - capacity = int64(cap) - return &csi.GetCapacityResponse{ - AvailableCapacity: capacity, + // Maximum volume size works better for capacity-aware + // pod scheduling than the available size. The other + // capacity values are available as metric. + AvailableCapacity: int64(cap.MaxVolumeSize), }, nil } diff --git a/pkg/pmem-csi-driver/main.go b/pkg/pmem-csi-driver/main.go index 7d686070ba..2e5c6d455d 100644 --- a/pkg/pmem-csi-driver/main.go +++ b/pkg/pmem-csi-driver/main.go @@ -33,27 +33,28 @@ func init() { flag.StringVar(&config.DriverName, "drivername", "pmem-csi.intel.com", "name of the driver") flag.StringVar(&config.NodeID, "nodeid", "nodeid", "node id") flag.StringVar(&config.Endpoint, "endpoint", "unix:///tmp/pmem-csi.sock", "PMEM CSI endpoint") - flag.BoolVar(&config.TestEndpoint, "testEndpoint", false, "also expose controller interface via endpoint (for testing only)") + flag.StringVar(&config.RegistryEndpoint, "registryEndpoint", "tcp://pmem-csi-controller:10000", "endpoint for internal registry server (controller listens, node connects)") flag.Var(&config.Mode, "mode", "driver run mode: controller or node") - flag.StringVar(&config.RegistryEndpoint, "registryEndpoint", "", "endpoint to connect/listen registry server") flag.StringVar(&config.CAFile, "caFile", "", "Root CA certificate file to use for verifying connections") flag.StringVar(&config.CertFile, "certFile", "", "SSL certificate file to use for authenticating client connections(RegistryServer/NodeControllerServer)") flag.StringVar(&config.KeyFile, "keyFile", "", "Private key file associated to certificate") flag.StringVar(&config.ClientCertFile, "clientCertFile", "", "Client SSL certificate file to use for authenticating peer connections, defaults to 'certFile'") flag.StringVar(&config.ClientKeyFile, "clientKeyFile", "", "Client private key associated to client certificate, defaults to 'keyFile'") - flag.UintVar(&config.PmemPercentage, "pmemPercentage", 100, "percentage of space to be used by the driver in each PMEM region") - /* Node mode options */ - flag.StringVar(&config.ControllerEndpoint, "controllerEndpoint", "", "internal node controller endpoint") - flag.Var(&config.DeviceManager, "deviceManager", "device manager to use to manage pmem devices, supported types: 'lvm' or 'direct' (= 'ndctl')") - flag.StringVar(&config.StateBasePath, "statePath", "", "Directory path where to persist the state of the driver running on a node, defaults to /var/lib/") - - /* scheduler options */ - flag.StringVar(&config.schedulerListen, "schedulerListen", "", "listen address (like :8000) for scheduler extender and mutating webhook, disabled by default") /* metrics options */ flag.StringVar(&config.metricsListen, "metricsListen", "", "listen address (like :8001) for prometheus metrics endpoint, disabled by default") flag.StringVar(&config.metricsPath, "metricsPath", "/metrics", "The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.") + /* Controller mode options */ + flag.StringVar(&config.schedulerListen, "schedulerListen", "", "controller: listen address (like :8000) for scheduler extender and mutating webhook, disabled by default") + + /* Node mode options */ + flag.StringVar(&config.ControllerEndpoint, "controllerEndpoint", "tcp://:10001", "node: internal node controller endpoint") + flag.BoolVar(&config.TestEndpoint, "testEndpoint", false, "node: also expose controller interface via CSI endpoint (for testing only)") + flag.Var(&config.DeviceManager, "deviceManager", "node: device manager to use to manage pmem devices, supported types: 'lvm' or 'direct' (= 'ndctl')") + flag.StringVar(&config.StateBasePath, "statePath", "", "node: directory path where to persist the state of the driver, defaults to /var/lib/") + flag.UintVar(&config.PmemPercentage, "pmemPercentage", 100, "node: percentage of space to be used by the driver in each PMEM region") + flag.Set("logtostderr", "true") } diff --git a/pkg/pmem-csi-driver/pmem-csi-driver.go b/pkg/pmem-csi-driver/pmem-csi-driver.go index 60f7c6f7d6..823205cd67 100644 --- a/pkg/pmem-csi-driver/pmem-csi-driver.go +++ b/pkg/pmem-csi-driver/pmem-csi-driver.go @@ -28,6 +28,7 @@ import ( pmemstate "github.com/intel/pmem-csi/pkg/pmem-state" "github.com/intel/pmem-csi/pkg/registryserver" "github.com/intel/pmem-csi/pkg/scheduler" + "github.com/kubernetes-csi/csi-lib-utils/metrics" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -63,6 +64,8 @@ func (mode *DriverMode) String() string { return string(*mode) } +// The mode strings are part of the metrics API (-> csi_controller, +// csi_node as subsystem), do not change them! const ( //Controller definition for controller driver mode Controller DriverMode = "controller" @@ -83,6 +86,27 @@ var ( }, []string{"version"}, ) + + pmemMaxDesc = prometheus.NewDesc( + "pmem_amount_max_volume_size", + "The size of the largest PMEM volume that can be created.", + nil, nil, + ) + pmemAvailableDesc = prometheus.NewDesc( + "pmem_amount_available", + "Remaining amount of PMEM on the host that can be used for new volumes.", + nil, nil, + ) + pmemManagedDesc = prometheus.NewDesc( + "pmem_amount_managed", + "Amount of PMEM on the host that is managed by PMEM-CSI.", + nil, nil, + ) + pmemTotalDesc = prometheus.NewDesc( + "pmem_amount_total", + "Total amount of PMEM on the host.", + nil, nil, + ) ) func init() { @@ -138,8 +162,50 @@ type csiDriver struct { cfg Config serverTLSConfig *tls.Config clientTLSConfig *tls.Config + gatherers prometheus.Gatherers } +// deviceManagerCollector is a wrapper around a PMEM device manager which +// takes GetCapacity values and turns them into metrics data. +type deviceManagerCollector struct { + pmdmanager.PmemDeviceManager +} + +// Describe implements prometheus.Collector.Describe. +func (dm deviceManagerCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(dm, ch) +} + +// Collect implements prometheus.Collector.Collect. +func (dm deviceManagerCollector) Collect(ch chan<- prometheus.Metric) { + capacity, err := dm.GetCapacity() + if err != nil { + return + } + ch <- prometheus.MustNewConstMetric( + pmemMaxDesc, + prometheus.GaugeValue, + float64(capacity.MaxVolumeSize), + ) + ch <- prometheus.MustNewConstMetric( + pmemAvailableDesc, + prometheus.GaugeValue, + float64(capacity.Available), + ) + ch <- prometheus.MustNewConstMetric( + pmemManagedDesc, + prometheus.GaugeValue, + float64(capacity.Managed), + ) + ch <- prometheus.MustNewConstMetric( + pmemTotalDesc, + prometheus.GaugeValue, + float64(capacity.Total), + ) +} + +var _ prometheus.Collector = deviceManagerCollector{} + func GetCSIDriver(cfg Config) (*csiDriver, error) { validModes := map[DriverMode]struct{}{ Controller: struct{}{}, @@ -152,19 +218,30 @@ func GetCSIDriver(cfg Config) (*csiDriver, error) { if _, ok := validModes[cfg.Mode]; !ok { return nil, fmt.Errorf("Invalid driver mode: %s", string(cfg.Mode)) } - if cfg.DriverName == "" || cfg.NodeID == "" || cfg.Endpoint == "" { - return nil, fmt.Errorf("One of mandatory(Drivername Node id or Endpoint) configuration option missing") + if cfg.DriverName == "" { + return nil, errors.New("driver name configuration option missing") } - if cfg.RegistryEndpoint == "" { - cfg.RegistryEndpoint = cfg.Endpoint + if cfg.Endpoint == "" { + return nil, errors.New("CSI endpoint configuration option missing") } - if cfg.ControllerEndpoint == "" { - cfg.ControllerEndpoint = cfg.Endpoint + if cfg.Mode == Node && cfg.NodeID == "" { + return nil, errors.New("node ID configuration option missing") + } + if cfg.Mode == Controller && cfg.RegistryEndpoint == "" { + return nil, errors.New("registry endpoint configuration option missing") + } + if cfg.Mode == Node && cfg.ControllerEndpoint == "" { + return nil, errors.New("internal controller endpoint configuration option missing") } - if cfg.Mode == Node && cfg.StateBasePath == "" { cfg.StateBasePath = "/var/lib/" + cfg.DriverName } + if cfg.Endpoint == cfg.RegistryEndpoint { + return nil, fmt.Errorf("CSI and registry endpoints must be different, both are: %q", cfg.Endpoint) + } + if cfg.Endpoint == cfg.ControllerEndpoint { + return nil, fmt.Errorf("CSI and internal control endpoints must be different, both are: %q", cfg.Endpoint) + } peerName := "pmem-registry" if cfg.Mode == Controller { @@ -203,6 +280,14 @@ func GetCSIDriver(cfg Config) (*csiDriver, error) { cfg: cfg, serverTLSConfig: serverConfig, clientTLSConfig: clientConfig, + // We use the default Prometheus registry here in addition to + // any custom CSIMetricsManager. Therefore we also return all + // data that is registered globally, including (but not + // limited to!) our own metrics data. For example, some Go + // runtime information + // (https://povilasv.me/prometheus-go-metrics/) are included, + // which may be useful. + gatherers: prometheus.Gatherers{prometheus.DefaultGatherer}, }, nil } @@ -222,34 +307,29 @@ func (csid *csiDriver) Run() error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if csid.cfg.Mode == Controller { - rs := registryserver.New(csid.clientTLSConfig) + // On the csi.sock endpoint we gather statistics for incoming + // CSI method calls like any other CSI driver. + cmm := metrics.NewCSIMetricsManagerForPlugin(csid.cfg.DriverName) + csid.gatherers = append(csid.gatherers, cmm.GetRegistry()) + + switch csid.cfg.Mode { + case Controller: + rs := registryserver.New(csid.clientTLSConfig, csid.cfg.DriverName) + csid.gatherers = append(csid.gatherers, rs.GetMetricsGatherer()) cs := NewMasterControllerServer(rs) - if csid.cfg.Endpoint != csid.cfg.RegistryEndpoint { - if err := s.Start(csid.cfg.Endpoint, nil, ids, cs); err != nil { - return err - } - if err := s.Start(csid.cfg.RegistryEndpoint, csid.serverTLSConfig, rs); err != nil { - return err - } - } else { - if err := s.Start(csid.cfg.Endpoint, csid.serverTLSConfig, ids, cs, rs); err != nil { - return err - } + if err := s.Start(csid.cfg.Endpoint, nil, cmm, ids, cs); err != nil { + return err + } + if err := s.Start(csid.cfg.RegistryEndpoint, csid.serverTLSConfig, nil /* no metrics gathering for registry at the moment */, rs); err != nil { + return err } // Also run scheduler extender? if _, err := csid.startScheduler(ctx, cancel, rs); err != nil { return err } - // And metrics server? - addr, err := csid.startMetrics(ctx, cancel) - if err != nil { - return err - } - klog.V(2).Infof("Prometheus endpoint started at https://%s%s", addr, csid.cfg.metricsPath) - } else if csid.cfg.Mode == Node { + case Node: dm, err := newDeviceManager(csid.cfg.DeviceManager, csid.cfg.PmemPercentage) if err != nil { return err @@ -261,32 +341,48 @@ func (csid *csiDriver) Run() error { cs := NewNodeControllerServer(csid.cfg.NodeID, dm, sm) ns := NewNodeServer(cs, filepath.Clean(csid.cfg.StateBasePath)+"/mount") - if csid.cfg.Endpoint != csid.cfg.ControllerEndpoint { - if err := s.Start(csid.cfg.ControllerEndpoint, csid.serverTLSConfig, cs); err != nil { - return err - } - if err := csid.registerNodeController(); err != nil { - return err - } - services := []grpcserver.Service{ids, ns} - if csid.cfg.TestEndpoint { - services = append(services, cs) - } - if err := s.Start(csid.cfg.Endpoint, nil, services...); err != nil { - return err - } - } else { - if err := s.Start(csid.cfg.Endpoint, nil, ids, cs, ns); err != nil { - return err - } - if err := csid.registerNodeController(); err != nil { - return err - } + // Internal CSI calls are tracked on the server side + // with a custom "pmem_csi_node" subsystem. The + // corresponding client calls use "pmem_csi_controller" with + // a tag that identifies the node that is being called. + cmmInternal := metrics.NewCSIMetricsManagerWithOptions(csid.cfg.DriverName, + metrics.WithSubsystem("pmem_csi_node"), + // Always add the instance label to allow correlating with + // the controller calls. + metrics.WithLabels(map[string]string{registryserver.NodeLabel: csid.cfg.NodeID}), + ) + csid.gatherers = append(csid.gatherers, cmmInternal.GetRegistry()) + if err := s.Start(csid.cfg.ControllerEndpoint, csid.serverTLSConfig, cmmInternal, cs); err != nil { + return err + } + if err := csid.registerNodeController(); err != nil { + return err + } + services := []grpcserver.Service{ids, ns} + if csid.cfg.TestEndpoint { + services = append(services, cs) + } + if err := s.Start(csid.cfg.Endpoint, nil, cmm, services...); err != nil { + return err } - } else { + + // Also collect metrics data via the device manager. + prometheus.WrapRegistererWith(prometheus.Labels{registryserver.NodeLabel: csid.cfg.NodeID}, prometheus.DefaultRegisterer).MustRegister( + deviceManagerCollector{dm}, + ) + default: return fmt.Errorf("Unsupported device mode '%v", csid.cfg.Mode) } + // And metrics server? + if csid.cfg.metricsListen != "" { + addr, err := csid.startMetrics(ctx, cancel) + if err != nil { + return err + } + klog.V(2).Infof("Prometheus endpoint started at http://%s%s", addr, csid.cfg.metricsPath) + } + c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) select { @@ -385,33 +481,34 @@ func (csid *csiDriver) startScheduler(ctx context.Context, cancel func(), rs *re return "", fmt.Errorf("failed to sync informer for type %v", t) } } - return csid.startHTTPSServer(ctx, cancel, csid.cfg.schedulerListen, sched) + return csid.startHTTPSServer(ctx, cancel, csid.cfg.schedulerListen, sched, true /* TLS */) } // startMetrics starts the HTTPS server for the Prometheus endpoint, if one is configured. // Error handling is the same as for startScheduler. func (csid *csiDriver) startMetrics(ctx context.Context, cancel func()) (string, error) { - if csid.cfg.metricsListen == "" { - return "", nil - } - - // We use the default Prometheus handler here and thus return all data that - // is registered globally, including (but not limited to!) our own metrics - // data. For example, some Go runtime information (https://povilasv.me/prometheus-go-metrics/) - // are included, which may be useful. mux := http.NewServeMux() - mux.Handle(csid.cfg.metricsPath, promhttp.Handler()) - return csid.startHTTPSServer(ctx, cancel, csid.cfg.metricsListen, mux) + mux.Handle(csid.cfg.metricsPath, + promhttp.InstrumentMetricHandler( + prometheus.DefaultRegisterer, + promhttp.HandlerFor(csid.gatherers, promhttp.HandlerOpts{}), + ), + ) + return csid.startHTTPSServer(ctx, cancel, csid.cfg.metricsListen, mux, false /* no TLS */) } // startHTTPSServer contains the common logic for starting and // stopping an HTTPS server. Returns an error or the address that can // be used in Dial("tcp") to reach the server (useful for testing when // "listen" does not include a port). -func (csid *csiDriver) startHTTPSServer(ctx context.Context, cancel func(), listen string, handler http.Handler) (string, error) { - config, err := pmemgrpc.LoadServerTLS(csid.cfg.CAFile, csid.cfg.CertFile, csid.cfg.KeyFile, "") - if err != nil { - return "", fmt.Errorf("initialize HTTPS config: %v", err) +func (csid *csiDriver) startHTTPSServer(ctx context.Context, cancel func(), listen string, handler http.Handler, useTLS bool) (string, error) { + var config *tls.Config + if useTLS { + c, err := pmemgrpc.LoadServerTLS(csid.cfg.CAFile, csid.cfg.CertFile, csid.cfg.KeyFile, "") + if err != nil { + return "", fmt.Errorf("initialize HTTPS config: %v", err) + } + config = c } server := http.Server{ Addr: listen, @@ -429,9 +526,14 @@ func (csid *csiDriver) startHTTPSServer(ctx context.Context, cancel func(), list go func() { defer tcpListener.Close() - err := server.ServeTLS(listener, csid.cfg.CertFile, csid.cfg.KeyFile) + var err error + if useTLS { + err = server.ServeTLS(listener, csid.cfg.CertFile, csid.cfg.KeyFile) + } else { + err = server.Serve(listener) + } if err != http.ErrServerClosed { - klog.Errorf("%s HTTPS server error: %v", listen, err) + klog.Errorf("%s HTTP(S) server error: %v", listen, err) } // Also stop main thread. cancel() diff --git a/pkg/pmem-csi-driver/pmem-csi-driver_test.go b/pkg/pmem-csi-driver/pmem-csi-driver_test.go index fda4007982..f49b603ea7 100644 --- a/pkg/pmem-csi-driver/pmem-csi-driver_test.go +++ b/pkg/pmem-csi-driver/pmem-csi-driver_test.go @@ -15,7 +15,6 @@ import ( "os" "testing" - pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -51,16 +50,17 @@ build_info{version="foo-bar-test"} 1 t.Run(n, func(t *testing.T) { path := "/metrics2" pmemd, err := GetCSIDriver(Config{ - Mode: Controller, - DriverName: "pmem-csi", - NodeID: "testnode", - Endpoint: "unused", - Version: "foo-bar-test", - CAFile: caFile, - CertFile: certFile, - KeyFile: keyFile, - metricsPath: path, - metricsListen: "127.0.0.1:", // port allocated dynamically + Mode: Controller, + DriverName: "pmem-csi", + NodeID: "testnode", + Endpoint: "unused", + RegistryEndpoint: "unused2", + Version: "foo-bar-test", + CAFile: caFile, + CertFile: certFile, + KeyFile: keyFile, + metricsPath: path, + metricsListen: "127.0.0.1:", // port allocated dynamically }) require.NoError(t, err, "get PMEM-CSI driver") @@ -69,18 +69,12 @@ build_info{version="foo-bar-test"} 1 addr, err := pmemd.startMetrics(ctx, cancel) require.NoError(t, err, "start server") - // Anonymous client: it makes requests without identifying itself. - tlsConfig, err := pmemgrpc.LoadClientTLS(caFile, "", "", "") - require.NoError(t, err, "load client TLS") - - tr := &http.Transport{ - TLSClientConfig: tlsConfig, - } + tr := &http.Transport{} defer tr.CloseIdleConnections() client := &http.Client{ Transport: tr, } - url := fmt.Sprintf("https://%s%s%s", addr, path, c.path) + url := fmt.Sprintf("http://%s%s%s", addr, path, c.path) resp, err := client.Get(url) checkResponse(t, &c.response, resp, err, n) }) diff --git a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go index e95d9c6cd0..f318892196 100644 --- a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go +++ b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go @@ -32,9 +32,11 @@ import ( ) const ( - controllerServicePort = 10000 - controllerMetricsPort = 10010 - nodeControllerPort = 10001 + controllerServicePort = 10000 + controllerMetricsPort = 10010 + nodeControllerPort = 10001 + nodeMetricsPort = 10010 + provisionerMetricsPort = 10011 ) type PmemCSIDriver struct { @@ -339,7 +341,7 @@ func validateCertificates(caCert, regKey, regCert, ncKey, ncCert []byte) error { // start a registry server server := grpcserver.NewNonBlockingGRPCServer() path := path.Join(tmp, "socket") - if err := server.Start("unix://"+path, regCfg); err != nil { + if err := server.Start("unix://"+path, regCfg, nil); err != nil { return err } defer server.ForceStop() @@ -626,6 +628,9 @@ func (d *PmemCSIDriver) getControllerStatefulSet() *appsv1.StatefulSet { "app": d.GetHyphenedName() + "-controller", "pmem-csi.intel.com/webhook": "ignore", }), + Annotations: map[string]string{ + "pmem-csi.intel.com/scrape": "containers", + }, }, Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ @@ -720,6 +725,9 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet { "app": d.GetHyphenedName() + "-node", "pmem-csi.intel.com/webhook": "ignore", }), + Annotations: map[string]string{ + "pmem-csi.intel.com/scrape": "containers", + }, }, Spec: corev1.PodSpec{ NodeSelector: d.Spec.NodeSelector, @@ -815,12 +823,12 @@ func (d *PmemCSIDriver) getControllerCommand() []string { "-mode=controller", "-endpoint=unix:///csi/csi-controller.sock", fmt.Sprintf("-registryEndpoint=tcp://0.0.0.0:%d", controllerServicePort), - fmt.Sprintf("-metricsListen=:%d", controllerMetricsPort), "-nodeid=$(KUBE_NODE_NAME)", "-caFile=/certs/ca.crt", "-certFile=/certs/tls.crt", "-keyFile=/certs/tls.key", "-drivername=$(PMEM_CSI_DRIVER_NAME)", + fmt.Sprintf("-metricsListen=:%d", controllerMetricsPort), } } @@ -841,6 +849,7 @@ func (d *PmemCSIDriver) getNodeDriverCommand() []string { "-statePath=/var/lib/$(PMEM_CSI_DRIVER_NAME)", "-drivername=$(PMEM_CSI_DRIVER_NAME)", fmt.Sprintf("-pmemPercentage=%d", d.Spec.PMEMPercentage), + fmt.Sprintf("-metricsListen=:%d", nodeMetricsPort), } } @@ -884,6 +893,7 @@ func (d *PmemCSIDriver) getControllerContainer() corev1.Container { MountPath: "/tmp", }, }, + Ports: d.getMetricsPorts(controllerMetricsPort), Resources: *d.Spec.ControllerResources, TerminationMessagePath: "/tmp/termination-log", SecurityContext: &corev1.SecurityContext{ @@ -962,6 +972,7 @@ func (d *PmemCSIDriver) getNodeDriverContainer() corev1.Container { MountPropagation: &bidirectional, }, }, + Ports: d.getMetricsPorts(nodeMetricsPort), Resources: *d.Spec.NodeResources, SecurityContext: &corev1.SecurityContext{ Privileged: &true, @@ -986,6 +997,7 @@ func (d *PmemCSIDriver) getProvisionerContainer() corev1.Container { "--feature-gates=Topology=true", "--strict-topology=true", "--timeout=5m", + fmt.Sprintf("--metrics-address=:%d", provisionerMetricsPort), }, VolumeMounts: []corev1.VolumeMount{ { @@ -993,6 +1005,7 @@ func (d *PmemCSIDriver) getProvisionerContainer() corev1.Container { MountPath: "/csi", }, }, + Ports: d.getMetricsPorts(provisionerMetricsPort), Resources: *d.Spec.ControllerResources, SecurityContext: &corev1.SecurityContext{ ReadOnlyRootFilesystem: &true, @@ -1034,6 +1047,15 @@ func (d *PmemCSIDriver) getNodeRegistrarContainer() corev1.Container { } } +func (d *PmemCSIDriver) getMetricsPorts(port int32) []corev1.ContainerPort { + return []corev1.ContainerPort{ + { + Name: "metrics", + ContainerPort: port, + }, + } +} + func (d *PmemCSIDriver) getObjectMeta(name string, isClusterResource bool) metav1.ObjectMeta { meta := metav1.ObjectMeta{ Name: name, diff --git a/pkg/pmem-device-manager/pmd-lvm.go b/pkg/pmem-device-manager/pmd-lvm.go index 8925e0e107..f09ddc5835 100644 --- a/pkg/pmem-device-manager/pmd-lvm.go +++ b/pkg/pmem-device-manager/pmd-lvm.go @@ -97,10 +97,30 @@ type vgInfo struct { free uint64 } -func (lvm *pmemLvm) GetCapacity() (uint64, error) { +func (lvm *pmemLvm) GetCapacity() (capacity Capacity, err error) { lvmMutex.Lock() defer lvmMutex.Unlock() - return lvm.getCapacity() + + var vgs []vgInfo + vgs, err = getVolumeGroups(lvm.volumeGroups) + if err != nil { + return + } + + for _, vg := range vgs { + if vg.free > capacity.MaxVolumeSize { + // TODO: alignment? + capacity.MaxVolumeSize = vg.free + } + capacity.Available += vg.free + capacity.Managed += vg.size + capacity.Total, err = totalSize() + if err != nil { + return + } + } + + return capacity, nil } func (lvm *pmemLvm) CreateDevice(volumeId string, size uint64) error { @@ -263,22 +283,6 @@ func parseLVSOutput(output string) (map[string]*PmemDeviceInfo, error) { return devices, nil } -func (lvm *pmemLvm) getCapacity() (uint64, error) { - var capacity uint64 - vgs, err := getVolumeGroups(lvm.volumeGroups) - if err != nil { - return 0, err - } - - for _, vg := range vgs { - if vg.free > capacity { - capacity = vg.free - } - } - - return capacity, nil -} - func getVolumeGroups(groups []string) ([]vgInfo, error) { vgs := []vgInfo{} args := append(vgsArgs, groups...) diff --git a/pkg/pmem-device-manager/pmd-manager.go b/pkg/pmem-device-manager/pmd-manager.go index cbc5194961..1e5ef5cda4 100644 --- a/pkg/pmem-device-manager/pmd-manager.go +++ b/pkg/pmem-device-manager/pmd-manager.go @@ -40,13 +40,28 @@ type PmemDeviceInfo struct { Size uint64 } +// Capacity contains information about PMEM. All sizes count bytes. +type Capacity struct { + // MaxVolumeSize is the size of the largest volume that + // currently can be created, considering alignment and + // fragmentation. + MaxVolumeSize uint64 + // Available is the sum of all PMEM that could be used for + // volumes. + Available uint64 + // Managed is all PMEM that is managed by the driver. + Managed uint64 + // Total is all PMEM found by the driver. + Total uint64 +} + //PmemDeviceManager interface to manage the PMEM block devices type PmemDeviceManager interface { // GetName returns current device manager's operation mode GetMode() api.DeviceMode - // GetCapacity returns the available maximum capacity that can be assigned to a Device/Volume - GetCapacity() (uint64, error) + // GetCapacity returns information about local capacity. + GetCapacity() (Capacity, error) // CreateDevice creates a new block device with give name, size and namespace mode // Possible errors: ErrNotEnoughSpace, ErrInvalid, ErrDeviceExists diff --git a/pkg/pmem-device-manager/pmd-ndctl.go b/pkg/pmem-device-manager/pmd-ndctl.go index 339ad2393c..e522f4f4a8 100644 --- a/pkg/pmem-device-manager/pmd-ndctl.go +++ b/pkg/pmem-device-manager/pmd-ndctl.go @@ -58,19 +58,25 @@ func (pmem *pmemNdctl) GetMode() api.DeviceMode { return api.DeviceModeDirect } -func (pmem *pmemNdctl) GetCapacity() (uint64, error) { +func (pmem *pmemNdctl) GetCapacity() (capacity Capacity, err error) { ndctlMutex.Lock() defer ndctlMutex.Unlock() - ndctx, err := ndctl.NewContext() + var ndctx *ndctl.Context + ndctx, err = ndctl.NewContext() if err != nil { - return 0, err + return } defer ndctx.Free() - var capacity uint64 for _, bus := range ndctx.GetBuses() { - for _, r := range bus.ActiveRegions() { + for _, r := range bus.AllRegions() { + capacity.Total += r.Size() + // TODO: check type?! + if !r.Enabled() { + continue + } + realalign := ndctlAlign * r.InterleaveWays() available := r.MaxAvailableExtent() // align down, avoid claiming more than what we really can serve @@ -78,9 +84,11 @@ func (pmem *pmemNdctl) GetCapacity() (uint64, error) { available /= realalign available *= realalign klog.V(4).Infof("GetCapacity: available after realalign: %d", available) - if available > capacity { - capacity = available + if available > capacity.MaxVolumeSize { + capacity.MaxVolumeSize = available } + capacity.Available += available + capacity.Managed += r.Size() } } // TODO: we should maintain capacity when adding or subtracting @@ -213,3 +221,21 @@ func namespaceToPmemInfo(ns *ndctl.Namespace) *PmemDeviceInfo { Size: ns.Size(), } } + +// totalSize sums up all PMEM regions, regardless whether they are +// enabled and regardless of their mode. +func totalSize() (size uint64, err error) { + var ndctx *ndctl.Context + ndctx, err = ndctl.NewContext() + if err != nil { + return + } + defer ndctx.Free() + + for _, bus := range ndctx.GetBuses() { + for _, region := range bus.AllRegions() { + size += region.Size() + } + } + return +} diff --git a/pkg/pmem-grpc/grpc.go b/pkg/pmem-grpc/grpc.go index b19f4966b2..bbb0d30060 100644 --- a/pkg/pmem-grpc/grpc.go +++ b/pkg/pmem-grpc/grpc.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/kubernetes-csi/csi-lib-utils/connection" + "github.com/kubernetes-csi/csi-lib-utils/metrics" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" @@ -53,7 +55,7 @@ func Connect(endpoint string, tlsConfig *tls.Config, dialOptions ...grpc.DialOpt } //NewServer is a helper function to start a grpc server at given endpoint and uses provided tlsConfig -func NewServer(endpoint string, tlsConfig *tls.Config, opts ...grpc.ServerOption) (*grpc.Server, net.Listener, error) { +func NewServer(endpoint string, tlsConfig *tls.Config, csiMetricsManager metrics.CSIMetricsManager, opts ...grpc.ServerOption) (*grpc.Server, net.Listener, error) { proto, addr, err := parseEndpoint(endpoint) if err != nil { return nil, nil, err @@ -70,7 +72,14 @@ func NewServer(endpoint string, tlsConfig *tls.Config, opts ...grpc.ServerOption return nil, nil, err } - opts = append(opts, grpc.UnaryInterceptor(pmemcommon.LogGRPCServer)) + interceptors := []grpc.UnaryServerInterceptor{ + pmemcommon.LogGRPCServer, + } + if csiMetricsManager != nil { + interceptors = append(interceptors, + connection.ExtendedCSIMetricsManager{CSIMetricsManager: csiMetricsManager}.RecordMetricsServerInterceptor) + } + opts = append(opts, grpc.ChainUnaryInterceptor(interceptors...)) if tlsConfig != nil { opts = append(opts, grpc.Creds(credentials.NewTLS(tlsConfig))) } diff --git a/pkg/registryserver/registryserver.go b/pkg/registryserver/registryserver.go index 011d199d53..f48e0dad9c 100644 --- a/pkg/registryserver/registryserver.go +++ b/pkg/registryserver/registryserver.go @@ -4,9 +4,11 @@ import ( "crypto/tls" "fmt" "sync" + "time" pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" registry "github.com/intel/pmem-csi/pkg/pmem-registry" + "github.com/kubernetes-csi/csi-lib-utils/metrics" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "golang.org/x/net/context" @@ -17,6 +19,10 @@ import ( "k8s.io/utils/keymutex" ) +// NodeLabel is a label used for Prometheus which identifies the +// node that the controller talks to. +const NodeLabel = "node" + // RegistryListener is an interface for registry server change listeners // All the callbacks are called once after updating the in-memory // registry data. @@ -41,6 +47,11 @@ type RegistryServer struct { clientTLSConfig *tls.Config nodeClients map[string]*NodeInfo listeners map[RegistryListener]struct{} + + // All nodes share the same metrics manager, but log samples + // with a different "pmem_csi_node" label and thus get their + // own histogram. + cmm metrics.CSIMetricsManager } type NodeInfo struct { @@ -63,15 +74,23 @@ func init() { prometheus.MustRegister(pmemNodes) } -func New(tlsConfig *tls.Config) *RegistryServer { +func New(tlsConfig *tls.Config, driverName string) *RegistryServer { return &RegistryServer{ rpcMutex: keymutex.NewHashed(-1), clientTLSConfig: tlsConfig, nodeClients: map[string]*NodeInfo{}, listeners: map[RegistryListener]struct{}{}, + cmm: metrics.NewCSIMetricsManagerWithOptions(driverName, + metrics.WithSubsystem("pmem_csi_controller"), + metrics.WithLabelNames(NodeLabel), + ), } } +func (rs *RegistryServer) GetMetricsGatherer() prometheus.Gatherer { + return rs.cmm.GetRegistry() +} + func (rs *RegistryServer) RegisterService(rpcServer *grpc.Server) { registry.RegisterRegistryServer(rpcServer, rs) } @@ -97,7 +116,32 @@ func (rs *RegistryServer) ConnectToNodeController(nodeId string) (*grpc.ClientCo klog.V(3).Infof("Connecting to node controller: %s", nodeInfo.Endpoint) - return pmemgrpc.Connect(nodeInfo.Endpoint, rs.clientTLSConfig) + return pmemgrpc.Connect(nodeInfo.Endpoint, rs.clientTLSConfig, + grpc.WithUnaryInterceptor(func( + ctx context.Context, + method string, + req, reply interface{}, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption) error { + start := time.Now() + err := invoker(ctx, method, req, reply, cc, opts...) + duration := time.Since(start) + cmmv, err2 := rs.cmm.WithLabelValues( + map[string]string{NodeLabel: nodeId}, + ) + if err2 != nil { + klog.Errorf("CSI call metrics: set label %s value: %v", NodeLabel, err2) + } else { + cmmv.RecordMetrics( + method, /* operationName */ + err, /* operationErr */ + duration, /* operationDuration */ + ) + } + return err + }), + ) } func (rs *RegistryServer) AddListener(l RegistryListener) { diff --git a/pkg/registryserver/registryserver_test.go b/pkg/registryserver/registryserver_test.go index e36c87b354..d27e15bc93 100644 --- a/pkg/registryserver/registryserver_test.go +++ b/pkg/registryserver/registryserver_test.go @@ -53,7 +53,7 @@ var _ = Describe("pmem registry", func() { BeforeEach(func() { var err error - registryServer = registryserver.New(nil) + registryServer = registryserver.New(nil, "pmem-csi.intel.com") caFile := os.ExpandEnv("${TEST_WORK}/pmem-ca/ca.pem") certFile := os.ExpandEnv("${TEST_WORK}/pmem-ca/pmem-registry.pem") @@ -62,7 +62,7 @@ var _ = Describe("pmem registry", func() { Expect(err).NotTo(HaveOccurred()) nbServer = grpcserver.NewNonBlockingGRPCServer() - err = nbServer.Start(registryServerEndpoint, tlsConfig, registryServer) + err = nbServer.Start(registryServerEndpoint, tlsConfig, nil, registryServer) Expect(err).NotTo(HaveOccurred()) _, err = os.Stat(registryServerSocketFile) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/deploy/deploy.go b/test/e2e/deploy/deploy.go index 100dcedecd..a4a066319c 100644 --- a/test/e2e/deploy/deploy.go +++ b/test/e2e/deploy/deploy.go @@ -106,7 +106,7 @@ func WaitForPMEMDriver(c *Cluster, name, namespace string) (metricsURL string) { } // We can connect to it and get metrics data. - metricsURL = fmt.Sprintf("https://%s:%d/metrics", c.NodeIP(0), port) + metricsURL = fmt.Sprintf("http://%s:%d/metrics", c.NodeIP(0), port) client := &http.Client{ Transport: &tr, Timeout: timeout, diff --git a/test/e2e/metrics/metrics.go b/test/e2e/metrics/metrics.go index fb04672e67..748ce6490d 100644 --- a/test/e2e/metrics/metrics.go +++ b/test/e2e/metrics/metrics.go @@ -7,15 +7,20 @@ SPDX-License-Identifier: Apache-2.0 package gotests import ( + "bytes" + "context" "crypto/tls" "fmt" - "k8s.io/kubernetes/test/e2e/framework" "net/http" + "strings" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + "github.com/intel/pmem-csi/test/e2e/deploy" ) @@ -50,7 +55,86 @@ var _ = deploy.Describe("direct-testing", "direct-testing-metrics", "", func(d * It("works", func() { // WaitForPMEMDriver already verified that "version" - // is returned. More tests could be added here. + // is returned and that "pmem_nodes" is correct. + // Here we check metrics support of each pod (= annotations + + // metrics endpoint). + pods, err := f.ClientSet.CoreV1().Pods(d.Namespace).List(context.Background(), metav1.ListOptions{}) + framework.ExpectNoError(err, "list pods") + + // We cannot connect to the node port directly from + // outside of the cluster. Instead of setting up + // port-forwarding, we rely on having socat in the + // "testing" deployment. We just need to find one of + // those pods... + socatPods, err := f.ClientSet.CoreV1().Pods(d.Namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app in ( pmem-csi-node-testing )", + }) + framework.ExpectNoError(err, "list socat pods") + Expect(pods.Items).NotTo(BeEmpty(), "at least one socat pod should be running") + socatPod := socatPods.Items[0] + + numPods := 0 + for _, pod := range pods.Items { + if pod.Annotations["pmem-csi.intel.com/scrape"] != "containers" { + continue + } + numPods++ + + numPorts := 0 + for _, container := range pod.Spec.Containers { + for _, port := range container.Ports { + if port.Name == "metrics" { + numPorts++ + + ip := pod.Status.PodIP + portNum := port.ContainerPort + // We use HTTP 1.0 to prevent the server from sending the response in chunks. + // That way we don't need to decode the response... + request := fmt.Sprintf(`GET /metrics HTTP/1.0 +Host: %s:%d +User-Agent: e2e/1.0 +Accept: */* + +`, ip, portNum) + options := framework.ExecOptions{ + Command: []string{ + "socat", + "STDIO", + fmt.Sprintf("TCP:%s:%d", ip, portNum), + }, + Namespace: socatPod.Namespace, + PodName: socatPod.Name, + Stdin: bytes.NewBufferString(request), + CaptureStdout: true, + CaptureStderr: true, + } + stdout, stderr, err := f.ExecWithOptions(options) + framework.ExpectNoError(err, "command failed in namespace %s, pod %s:\nstderr:\n%s\nstdout:%s\n", + socatPod.Namespace, socatPod.Name, stderr, stdout) + name := pod.Name + "/" + container.Name + if strings.HasPrefix(container.Name, "pmem") { + Expect(stdout).To(ContainSubstring("go_threads "), name) + Expect(stdout).To(ContainSubstring("process_open_fds "), name) + Expect(stdout).To(ContainSubstring("csi_plugin_operations_seconds "), name) + if strings.HasPrefix(pod.Name, "pmem-csi-controller") { + Expect(stdout).To(ContainSubstring("pmem_nodes "), name) + Expect(stdout).To(ContainSubstring("pmem_csi_controller_operations_seconds "), name) + } else { + Expect(stdout).To(ContainSubstring("pmem_amount_available "), name) + Expect(stdout).To(ContainSubstring("pmem_amount_managed "), name) + Expect(stdout).To(ContainSubstring("pmem_amount_max_volume_size "), name) + Expect(stdout).To(ContainSubstring("pmem_amount_total "), name) + Expect(stdout).To(ContainSubstring("pmem_csi_node_operations_seconds "), name) + } + } else { + Expect(stdout).To(ContainSubstring("csi_sidecar_operations_seconds "), name) + } + } + } + } + Expect(numPorts).NotTo(Equal(0), "at least one container should have a 'metrics' port") + } + Expect(numPods).NotTo(Equal(0), "at least one pod should have a 'pmem-csi.intel.com/scrape: containers' annotation") }) It("rejects large headers", func() { diff --git a/test/e2e/operator/validate/validate.go b/test/e2e/operator/validate/validate.go index 70797e6ce0..28ae6ce080 100644 --- a/test/e2e/operator/validate/validate.go +++ b/test/e2e/operator/validate/validate.go @@ -248,6 +248,8 @@ func parseDefaultSpecValues() map[string]interface{} { terminationMessagePath: /dev/termination-log terminationMessagePolicy: File imagePullPolicy: IfNotPresent + ports: + protocol: TCP volumes: secret: defaultMode: 420` diff --git a/test/e2e/tls/tls.go b/test/e2e/tls/tls.go index e95011a361..1c9d270edf 100644 --- a/test/e2e/tls/tls.go +++ b/test/e2e/tls/tls.go @@ -172,8 +172,9 @@ func checkTLS(f *framework.Framework, server string) { ports := re.FindAllStringSubmatch(output, -1) for _, entry := range ports { port, ciphers := entry[1], entry[2] - if port == "10002" { - // The socat debugging port. Can be ignored. + switch port { + case "10002", "10010", "10011": + // The socat debugging port and metrics ports. Can be ignored. continue } // All other ports must use TLS, with exactly the