rely on docker to rotate logs for watchdog/job_exporter

pai-management/bootstrap/prometheus/node-exporter-ds.yaml.template

@@ -89,8 +89,12 @@ spec:
         readinessProbe:
           exec:
             command:
-            - python
-            - /usr/local/healthy_check.py
+            - "python"
+            - "/job_exporter/no_older_than.py"
+            - "--delta"
+            - "60"
+            - "/datastorage/prometheus/job_exporter.prom"
+            - "/datastorage/prometheus/gpu_exporter.prom"
           initialDelaySeconds: 30
           periodSeconds: 30
         resources:

pai-management/bootstrap/prometheus/watchdog-ds.yaml.template

@@ -40,6 +40,12 @@ spec:
       - name: watchdog
         image: {{ clusterinfo["dockerregistryinfo"]["prefix"] }}watchdog:{{ clusterinfo["dockerregistryinfo"]["docker_tag"] }}
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 9101
+          initialDelaySeconds: 30
+          periodSeconds: 30
         resources:
           limits:
             memory: "1Gi"
@@ -53,6 +59,8 @@ spec:
           - "/watchdog.py"
           - "--interval"
           - "30"
+          - "--port"
+          - "9101"
           - "{{ clusterinfo['webportalinfo']['k8s_api_server_uri'] }}"
       volumes:
         - name: collector-mount

pai-management/src/gpu-exporter/dockerfile

@@ -15,31 +15,19 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-FROM ubuntu:16.04
+FROM python:2.7
 
-#
-# Preparation
-#
 
 ENV NVIDIA_VERSION=current
 ENV NV_DRIVER=/var/drivers/nvidia/$NVIDIA_VERSION
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NV_DRIVER/lib:$NV_DRIVER/lib64
 ENV PATH=$PATH:$NV_DRIVER/bin
 
-WORKDIR /root/
-
-RUN apt-get update && \
-    apt-get -y install wget build-essential python python-pip git pciutils
-
-COPY copied_file/exporter/* /usr/local/
+RUN mkdir -p /job_exporter
+COPY copied_file/exporter/* /job_exporter/
 
 RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz
-RUN cp docker-17.06.2-ce.tgz /usr/local
-RUN tar xzvf /usr/local/docker-17.06.2-ce.tgz -C /usr/local/
+RUN tar xzvf docker-17.06.2-ce.tgz -C /usr/local/
 RUN cp -r /usr/local/docker/* /usr/bin/
 
-#
-# start
-#
-
-CMD python /usr/local/job_exporter.py /datastorage/prometheus 30
+CMD python /job_exporter/job_exporter.py /datastorage/prometheus 30

prometheus/exporter/job_exporter.py

@@ -20,7 +20,6 @@
 import sys
 import time
 import logging
-from logging.handlers import RotatingFileHandler
 
 import docker_stats
 import docker_inspect
@@ -43,7 +42,6 @@ def parse_from_labels(labels):
         else:
             otherLabels[key] = val
 
-
     return gpuIds, otherLabels
 
 
@@ -88,14 +86,6 @@ def main(argv):
     jobMetricsPath = logDir + "/job_exporter.prom"
     timeSleep = int(argv[1])
 
-    rootLogger = logging.getLogger()
-    rootLogger.setLevel(logging.INFO)
-    fh = RotatingFileHandler(logDir + "/gpu_exporter.log", maxBytes= 1024 * 1024 * 10, backupCount=5)
-    fh.setLevel(logging.INFO)
-    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s")
-    fh.setFormatter(formatter)
-    rootLogger.addHandler(fh)
-
     iter = 0
 
     singleton = utils.Singleton(gpu_exporter.collect_gpu_info)
@@ -121,4 +111,7 @@ def main(argv):
 
 
 if __name__ == "__main__":
+    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s",
+            level=logging.INFO)
+
     main(sys.argv[1:])

prometheus/exporter/no_older_than.py

@@ -0,0 +1,42 @@
+#!/usr/bin/python
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import argparse
+import datetime
+import os
+
+def check_no_older_than(paths, delta):
+    """ raise RuntimeError exception if any path in paths is older than `now - delta` """
+    now = datetime.datetime.now()
+    delta = datetime.timedelta(seconds=delta)
+    oldest = now - delta
+
+    for path in paths:
+        mtime = os.path.getmtime(path)
+        mtime = datetime.datetime.fromtimestamp(mtime)
+        if oldest > mtime:
+            raise RuntimeError("{} was updated more than {} seconds ago".format(path, delta))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("paths", nargs="+", help="file to be checked")
+    parser.add_argument("-d", "--delta", type=int, default=60, help="check file is no older than -d seconds")
+    args = parser.parse_args()
+
+    check_no_older_than(args.paths, args.delta)

prometheus/exporter/watchdog.py

@@ -361,14 +361,6 @@ def main(args):
 
     hosts = load_machine_list(args.hosts)
 
-    rootLogger = logging.getLogger()
-    rootLogger.setLevel(logging.INFO)
-    fh = RotatingFileHandler(logDir + "/watchdog.log", maxBytes= 1024 * 1024 * 100, backupCount=5)
-    fh.setLevel(logging.INFO)
-    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s")
-    fh.setFormatter(formatter)
-    rootLogger.addHandler(fh)
-
     list_pods_url = "{}/api/v1/namespaces/default/pods/".format(address)
     list_nodes_url = "{}/api/v1/nodes/".format(address)
 
@@ -423,4 +415,7 @@ def main(args):
     parser.add_argument("--hosts", "-m", help="yaml file path contains host info", default="/etc/watchdog/config.yml")
     args = parser.parse_args()
 
+    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s",
+            level=logging.INFO)
+
     main(args)