Clean jobs containers when delete NM (#1296)

clean job when delete nm
microsoft · Sep 7, 2018 · 11fd7df · 11fd7df
1 parent 19f68a0
commit 11fd7df
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 2 deletions.
diff --git a/pai-management/bootstrap/hadoop-node-manager/delete.yaml.template b/pai-management/bootstrap/hadoop-node-manager/delete.yaml.template
@@ -39,11 +39,13 @@ spec:
           name: data-path
         - mountPath: /hadoop-node-manager-delete
           name: hadoop-node-manager-delete-config
+        - mountPath: /var/run/docker.sock
+          name: docker-socket
         env:
         - name: DELETE_CONFIG
           value: hadoop-node-manager-delete
         - name: WORKER_CONFIG
-          value: delete-data.sh
+          value: delete.sh
         readinessProbe:
           exec:
             command:
@@ -60,3 +62,7 @@ spec:
       - name: hadoop-node-manager-delete-config
         configMap:
           name: hadoop-node-manager-delete
+      - name: docker-socket
+        hostPath:
+          path: /var/run/docker.sock
+
diff --git a/...hadoop-node-manager-delete/delete-data.sh → ...ager/hadoop-node-manager-delete/delete.sh b/...hadoop-node-manager-delete/delete-data.sh → ...ager/hadoop-node-manager-delete/delete.sh
@@ -17,6 +17,19 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+# Clean running job
+
+if which docker > /dev/null && [ -S /var/run/docker.sock ]; then
+
+    echo "Clean hadoop jobs"
+
+    docker ps | awk '/container_\w{3}_[0-9]{13}_[0-9]{4}_[0-9]{2}_[0-9]{6}/ { print $NF}' | xargs timeout 30 docker stop || \
+    docker ps | awk '/container_\w{3}_[0-9]{13}_[0-9]{4}_[0-9]{2}_[0-9]{6}/ { print $NF}' | xargs docker kill
+
+fi
+
+
+# Clean data
 
 echo "Clean the hadoop node manager's data on the disk"
 

diff --git a/pai-management/src/cleaning-image/dockerfile b/pai-management/src/cleaning-image/dockerfile
@@ -17,7 +17,12 @@
 
 FROM base-image
 
+RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz && \
+    tar xzvf docker-17.06.2-ce.tgz && \
+    mv docker/* /usr/bin/ && \
+    rm docker-17.06.2-ce.tgz
+
 COPY start.sh /usr/local/start.sh
 RUN chmod a+x /usr/local/start.sh
 
-CMD ["/usr/local/start.sh"]
+CMD ["/usr/local/start.sh"]