[docker_daemon] Add metadata, tests and fix missing yaml option

checks.d/docker_daemon.py

@@ -9,6 +9,7 @@
 import socket
 import urllib2
 from collections import defaultdict, Counter, deque
+from math import ceil
 
 # project
 from checks import AgentCheck
@@ -21,7 +22,6 @@
 
 EVENT_TYPE = 'docker'
 SERVICE_CHECK_NAME = 'docker.service_up'
-DISK_METRICS_PREFIX = 'docker.disk.{0}'
 SIZE_REFRESH_RATE = 5  # Collect container sizes every 5 iterations of the check
 MAX_CGROUP_LISTING_RETRIES = 3
 CONTAINER_ID_RE = re.compile('[0-9a-f]{64}')
@@ -428,7 +428,6 @@ def _filter_containers(self, containers):
                 self._filtered_containers.add(container_name)
                 self.log.debug("Container {0} is filtered".format(container["Names"][0]))
 
-
     def _are_tags_filtered(self, tags):
         if self._tags_match_patterns(tags, self._exclude_patterns):
             if self._tags_match_patterns(tags, self._include_patterns):
@@ -461,7 +460,7 @@ def _report_container_size(self, containers_by_id):
             if "SizeRw" in container:
 
                 m_func(self, 'docker.container.size_rw', container['SizeRw'],
-                    tags=tags)
+                       tags=tags)
             if "SizeRootFs" in container:
                 m_func(
                     self, 'docker.container.size_rootfs', container['SizeRootFs'],
@@ -651,24 +650,42 @@ def _format_events(self, aggregated_events, containers_by_id):
     def _report_disk_stats(self):
         """Report metrics about the volume space usage"""
         stats = {
-            'used': None,
-            'total': None,
-            'free': None
+            'docker.data.used': None,
+            'docker.data.total': None,
+            'docker.data.free': None,
+            'docker.metadata.used': None,
+            'docker.metadata.total': None,
+            'docker.metadata.free': None
+            # these two are calculated by _calc_percent_disk_stats
+            # 'docker.data.percent': None,
+            # 'docker.metadata.percent': None
         }
         info = self.docker_client.info()
         driver_status = info.get('DriverStatus', [])
+        if not driver_status:
+            self.log.warning('Disk metrics collection is enabled but docker info did not'
+                             ' report any. Your storage driver might not support them, skipping.')
+            return
         for metric in driver_status:
-            if metric[0] == 'Data Space Used':
-                stats['used'] = metric[1]
-            elif metric[0] == 'Data Space Total':
-                stats['total'] = metric[1]
-            elif metric[0] == 'Data Space Available':
-                stats['free'] = metric[1]
+            # only consider metrics about disk space
+            if len(metric) == 2 and 'Space' in metric[0]:
+                # identify Data and Metadata metrics
+                mtype = 'data'
+                if 'Metadata' in metric[0]:
+                    mtype = 'metadata'
+
+                if 'Used' in metric[0]:
+                    stats['docker.{0}.used'.format(mtype)] = metric[1]
+                elif 'Space Total' in metric[0]:
+                    stats['docker.{0}.total'.format(mtype)] = metric[1]
+                elif 'Space Available' in metric[0]:
+                    stats['docker.{0}.free'.format(mtype)] = metric[1]
         stats = self._format_disk_metrics(stats)
+        stats.update(self._calc_percent_disk_stats(stats))
         tags = self._get_tags()
         for name, val in stats.iteritems():
-            if val:
-                self.gauge(DISK_METRICS_PREFIX.format(name), val, tags)
+            if val is not None:
+                self.gauge(name, val, tags)
 
     def _format_disk_metrics(self, metrics):
         """Cast the disk stats to float and convert them to bytes"""
@@ -682,8 +699,31 @@ def _format_disk_metrics(self, metrics):
                     metrics[name] = val
                 except KeyError:
                     self.log.error('Unrecognized unit %s for disk metric %s. Dropping it.' % (unit, name))
+                    metrics[name] = None
         return metrics
 
+    def _calc_percent_disk_stats(self, stats):
+        """Calculate a percentage of used disk space for data and metadata"""
+        mtypes = ['data', 'metadata']
+        percs = {}
+        for mtype in mtypes:
+            used = stats.get('docker.{0}.used'.format(mtype))
+            total = stats.get('docker.{0}.total'.format(mtype))
+            free = stats.get('docker.{0}.free'.format(mtype))
+            if used and total and free and ceil(total) < free + used:
+                self.log.error('used, free, and total disk metrics may be wrong, '
+                               'unable to calculate percentage.')
+                return {}
+            try:
+                if isinstance(used, int):
+                    percs['docker.{0}.percent'.format(mtype)] = round(100 * float(used) / float(total), 2)
+                elif isinstance(free, int):
+                    percs['docker.{0}.percent'.format(mtype)] = round(100 * (1.0 - (float(free) / float(total))), 2)
+            except ZeroDivisionError:
+                self.log.error('docker.{0}.total is 0, calculating docker.{1}.percent'
+                               ' is not possible.'.format(mtype, mtype))
+        return percs
+
     # Cgroups
 
     def _get_cgroup_file(self, cgroup, container_id, filename):

conf.d/docker_daemon.yaml.example

@@ -58,6 +58,16 @@ instances:
     #
     # collect_image_size: false
 
+    # Collect disk metrics (total, used, free) through the docker info command for data and metadata.
+    # This is useful when these values can't be obtained by the disk check.
+    # Example: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
+    # Note that it only works when the storage driver is devicemapper.
+    # Explanation of these metrics can be found here:
+    # https://github.com/docker/docker/blob/v1.11.1/daemon/graphdriver/devmapper/README.md
+    # Defaults to false.
+    #
+    # collect_disk_stats: true
+
 
     # Exclude containers based on their tags
     # An excluded container will be completely ignored. The rule is a regex on the tags.
@@ -112,4 +122,4 @@ instances:
     # List of container label names that should be collected and sent as tags.
     # Default to None
     # Example:
-    # collect_labels_as_tags: ["com.docker.compose.service", "com.docker.compose.project"] 
+    # collect_labels_as_tags: ["com.docker.compose.service", "com.docker.compose.project"]