Integrate Dshuttle into PAI

src/dshuttle-csi/deploy/delete.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+pushd $(dirname "$0") > /dev/null
+
+echo "Call stop script to stop all service first"
+/bin/bash stop.sh || exit $?
+
+
+popd > /dev/null

src/dshuttle-csi/deploy/dshuttle-csi-daemon.yaml.template

@@ -0,0 +1,143 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: dshuttle-csi-daemon
+spec:
+  selector:
+    matchLabels:
+      app: dshuttle-csi-daemon
+  template:
+    metadata:
+      labels:
+        app: dshuttle-csi-daemon
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      initContainers:
+      # This change aimed to avoid OS build file index for alluxio-fuse-fs, which will consume much disk space
+      - name: change-updatedb-conf
+        image: dshuttle.azurecr.io/dshuttle/dshuttle-csi:25037dc
+        imagePullPolicy: Always
+        securityContext:
+          runAsUser: 0
+        command: ["/bin/bash", "-c"]
+        args:
+        - FILE=/host-config/updatedb.conf && grep -q 'PRUNEFS=".*fuse.alluxio-fuse.*"' "$FILE" || echo $(sed  '/PRUNEFS=/s/"$/ fuse.alluxio-fuse"/' "$FILE") > $FILE
+        volumeMounts:
+        - name: etc
+          mountPath: /host-config/updatedb.conf
+          subPath: updatedb.conf
+      containers:
+      - name: node-driver-registrar
+        image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "rm -rf /registration/dshuttle-reg.sock /var/lib/kubelet/plugins/csi-dshuttle-plugin"]
+        args:
+        - --v=5
+        - --csi-address=/plugin/csi.sock
+        - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-dshuttle-plugin/csi.sock
+        env:
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /plugin
+        - name: registration-dir
+          mountPath: /registration
+      - name: dshuttle-csi-daemon
+        securityContext:
+          privileged: true
+          runAsUser: 0
+        image: dshuttle.azurecr.io/dshuttle/dshuttle-csi:25037dc
+        command: ["/usr/local/bin/dshuttle-csi"]
+        args :
+        - "--v=4"
+        - "--nodeid=$(NODE_ID)"
+        - "--endpoint=$(CSI_ENDPOINT)"
+        env:
+        - name: ALLUXIO_CLIENT_HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: ALLUXIO_CLIENT_JAVA_OPTS
+          value: " -Dalluxio.user.hostname=$(ALLUXIO_CLIENT_HOSTNAME) -Dalluxio.worker.hostname=$(ALLUXIO_CLIENT_HOSTNAME) "
+        - name: NODE_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: CSI_ENDPOINT
+          value: unix://plugin/csi.sock
+        envFrom:
+        - configMapRef:
+            name: dshuttle-config
+        {%- if cluster_cfg['cluster']['common']['qos-switch'] == "true" %}
+        resources:
+          limits:
+            memory: "{{ cluster_cfg['dshuttle']['csi_daemon_limit_mem'] }}"
+          requests:
+            memory: "{{ cluster_cfg['dshuttle']['csi_daemon_request_mem'] }}"
+        {%- endif %}
+        imagePullPolicy: "Always"
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /plugin
+        - name: pods-mount-dir
+          mountPath: /var/lib/kubelet/pods
+          mountPropagation: "Bidirectional"
+        - name: dshuttle-domain
+          mountPath: /opt/domain
+        - name: fuse-logs
+          mountPath: /opt/alluxio/logs
+        - name: dshuttle-log-config
+          mountPath: /opt/alluxio/conf/log4j.properties
+          subPath: log4j.properties
+      volumes:
+      - name: plugin-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins/csi-dshuttle-plugin
+          type: DirectoryOrCreate
+      - name: pods-mount-dir
+        hostPath:
+          path: /var/lib/kubelet/pods
+          type: Directory
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry
+          type: Directory
+        name: registration-dir
+      - name: dshuttle-domain
+        hostPath:
+          path: /tmp/alluxio-domain
+          type: "Directory"
+      - name: fuse-logs
+        hostPath:
+          path: /var/log/dshuttle
+          type: DirectoryOrCreate
+      - name: dshuttle-log-config
+        configMap:
+          name: dshuttle-log-config
+      - name: etc
+        hostPath:
+          path: /etc
+      imagePullSecrets:
+      - name: dshuttle-regcred

src/dshuttle-csi/deploy/dshuttle-csi-driver.yaml

@@ -0,0 +1,24 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: dshuttle
+spec:
+  attachRequired: false
+  podInfoOnMount: true

src/dshuttle-csi/deploy/service.yaml

@@ -0,0 +1,36 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+cluster-type:
+  - k8s
+
+prerequisite:
+  - cluster-configuration
+  - dshuttle-master
+  - dshuttle-worker
+
+template-list:
+  - dshuttle-csi-daemon.yaml
+  - start.sh
+  - stop.sh
+
+start-script: start.sh
+stop-script: stop.sh
+delete-script: delete.sh
+
+deploy-rules:
+  - in: pai-worker

src/dshuttle-csi/deploy/start.sh.template

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+{%- if cluster_cfg['cluster']['common']['dshuttle'] == 'true' %}
+pushd $(dirname "$0") > /dev/null
+
+kubectl apply --overwrite=true -f dshuttle-csi-driver.yaml || exit $?
+kubectl apply --overwrite=true -f dshuttle-csi-daemon.yaml || exit $?
+
+sleep 10
+# Wait until the service is ready.
+PYTHONPATH="../../../deployment" python -m k8sPaiLibrary.monitorTool.check_pod_ready_status -w -k app -v dshuttle-csi-daemon || exit $?
+
+popd > /dev/null
+{%- endif %}

src/dshuttle-csi/deploy/stop.sh.template

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+{%- if cluster_cfg['cluster']['common']['dshuttle'] == 'true' %}
+pushd $(dirname "$0") > /dev/null
+
+if kubectl get daemonset | grep -q "dshuttle-csi-daemon"; then
+    kubectl delete daemonset dshuttle-csi-daemon || exit $?
+fi
+
+if kubectl get csidriver | grep -q "dshuttle"; then
+    kubectl delete csidriver dshuttle || exit $?
+fi
+
+popd > /dev/null
+{%- endif %}

src/dshuttle-master/deploy/delete.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+pushd $(dirname "$0") > /dev/null
+
+echo "Call stop to stop all dshuttle-master pod first"
+/bin/bash stop.sh || exit $?
+
+echo "Create dshuttle-master-delete configmap for deleting data on the host"
+kubectl create configmap dshuttle-master-delete --from-file=dshuttle-master-delete/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $?
+
+echo "Create cleaner daemon"
+kubectl apply --overwrite=true -f delete.yaml || exit $?
+sleep 5
+
+PYTHONPATH="../../../deployment" python -m  k8sPaiLibrary.monitorTool.check_pod_ready_status -w -k app -v delete-batch-job-dshuttle-master || exit $?
+
+echo "Dshuttle master clean job is done"
+echo "Delete dshuttle master cleaner daemon and configmap"
+if kubectl get daemonset | grep -q "delete-batch-job-dshuttle-master"; then
+    kubectl delete ds delete-batch-job-dshuttle-master || exit $?
+fi
+
+if kubectl get configmap | grep -q "dshuttle-master-delete"; then
+    kubectl delete configmap dshuttle-master-delete || exit $?
+fi
+sleep 5
+
+popd > /dev/null