Add metadata, testing and fix missing yaml option

checks.d/docker_daemon.py

@@ -428,7 +428,6 @@ def _filter_containers(self, containers):
                 self._filtered_containers.add(container_name)
                 self.log.debug("Container {0} is filtered".format(container["Names"][0]))
 
-
     def _are_tags_filtered(self, tags):
         if self._tags_match_patterns(tags, self._exclude_patterns):
             if self._tags_match_patterns(tags, self._include_patterns):
@@ -461,7 +460,7 @@ def _report_container_size(self, containers_by_id):
             if "SizeRw" in container:
 
                 m_func(self, 'docker.container.size_rw', container['SizeRw'],
-                    tags=tags)
+                       tags=tags)
             if "SizeRootFs" in container:
                 m_func(
                     self, 'docker.container.size_rootfs', container['SizeRootFs'],
@@ -651,24 +650,38 @@ def _format_events(self, aggregated_events, containers_by_id):
     def _report_disk_stats(self):
         """Report metrics about the volume space usage"""
         stats = {
-            'used': None,
-            'total': None,
-            'free': None
+            'docker.data.used': 0.0,
+            'docker.data.total': 0.0,
+            'docker.data.free': 0.0,
+            'docker.metadata.used': 0.0,
+            'docker.metadata.total': 0.0,
+            'docker.metadata.free': 0.0
+            # these two are calculated by _calc_percent_disk_stats
+            # 'docker.data.percent': 0.0,
+            # 'docker.metadata.percent': 0.0
         }
         info = self.docker_client.info()
         driver_status = info.get('DriverStatus', [])
         for metric in driver_status:
-            if metric[0] == 'Data Space Used':
-                stats['used'] = metric[1]
-            elif metric[0] == 'Data Space Total':
-                stats['total'] = metric[1]
-            elif metric[0] == 'Data Space Available':
-                stats['free'] = metric[1]
+            # only consider metrics about disk space
+            if len(metric) == 2 and 'Space' in metric[0]:
+                # identify Data and Metadata metrics
+                mtype = 'data'
+                if 'Metadata' in metric[0]:
+                    mtype = 'metadata'
+
+                if 'Used' in metric[0]:
+                    stats['docker.{0}.used'.format(mtype)] = metric[1]
+                elif 'Space Total' in metric[0]:
+                    stats['docker.{0}.total'.format(mtype)] = metric[1]
+                elif 'Space Available' in metric[0]:
+                    stats['docker.{0}.free'.format(mtype)] = metric[1]
         stats = self._format_disk_metrics(stats)
+        stats.update(self._calc_percent_disk_stats(stats))
         tags = self._get_tags()
         for name, val in stats.iteritems():
             if val:
-                self.gauge(DISK_METRICS_PREFIX.format(name), val, tags)
+                self.gauge(name, val, tags)
 
     def _format_disk_metrics(self, metrics):
         """Cast the disk stats to float and convert them to bytes"""
@@ -684,6 +697,24 @@ def _format_disk_metrics(self, metrics):
                     self.log.error('Unrecognized unit %s for disk metric %s. Dropping it.' % (unit, name))
         return metrics
 
+    def _calc_percent_disk_stats(self, stats):
+        """Calculate a percentage of used disk space for data and metadata"""
+        mtypes = ['data', 'metadata']
+        percs = {}
+        for mtype in mtypes:
+            used = stats.get('docker.{0}.used'.format(mtype))
+            total = stats.get('docker.{0}.total'.format(mtype))
+            free = stats.get('docker.{0}.free'.format(mtype))
+            if total < free + used:
+                self.log.error('used, free, and total disk metrics may be wrong, '
+                               'unable to calculate percentage.')
+                return {}
+            if used > 0.0 and total > 0.0:
+                percs['docker.{0}.percent'.format(mtype)] = round(100 * float(used) / float(total), 2)
+            elif free > 0.0 and total > 0.0:
+                percs['docker.{0}.percent'.format(mtype)] = round(100 * (1.0 - (float(free) / float(total))), 2)
+        return percs
+
     # Cgroups
 
     def _get_cgroup_file(self, cgroup, container_id, filename):

conf.d/docker_daemon.yaml.example

@@ -58,6 +58,16 @@ instances:
     #
     # collect_image_size: false
 
+    # Collect disk metrics (total, used, free) through the docker info command for data and metadata.
+    # This is useful when these values can't be obtained by the disk check.
+    # Example: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
+    # Note that it only works when the storage driver is devicemapper.
+    # Explanation of these metrics can be found here:
+    # https://github.com/docker/docker/blob/v1.11.1/daemon/graphdriver/devmapper/README.md
+    # Defaults to false.
+    #
+    # collect_disk_stats: true
+
 
     # Exclude containers based on their tags
     # An excluded container will be completely ignored. The rule is a regex on the tags.
@@ -112,4 +122,4 @@ instances:
     # List of container label names that should be collected and sent as tags.
     # Default to None
     # Example:
-    # collect_labels_as_tags: ["com.docker.compose.service", "com.docker.compose.project"] 
+    # collect_labels_as_tags: ["com.docker.compose.service", "com.docker.compose.project"]

tests/checks/mock/test_docker_daemon.py

@@ -0,0 +1,106 @@
+# stdlib
+import mock
+
+# project
+from tests.checks.common import AgentCheckTest
+
+MOCK_CONFIG = {
+    "init_config": {},
+    "instances": [{
+        "url": "unix://var/run/docker.sock",
+        "collect_disk_stats": True,
+    }]
+}
+
+
+class TestCheckDockerDaemon(AgentCheckTest):
+    CHECK_NAME = 'docker_daemon'
+
+    def mock_normal_get_info(self):
+        return {
+            'DriverStatus': [
+                ['Data Space Used', '1 GB'],
+                ['Data Space Available', '9 GB'],
+                ['Data Space Total', '10 GB'],
+                ['Metadata Space Used', '1 MB'],
+                ['Metadata Space Available', '9 MB'],
+                ['Metadata Space Total', '10 MB'],
+            ]
+        }
+
+    def mock_get_info_no_used(self):
+        return {
+            'DriverStatus': [
+                ['Data Space Available', '9 GB'],
+                ['Data Space Total', '10 GB'],
+                ['Metadata Space Available', '9 MB'],
+                ['Metadata Space Total', '10 MB'],
+            ]
+        }
+
+    def mock_get_info_no_data(self):
+        return {
+            'DriverStatus': [
+                ['Metadata Space Available', '9 MB'],
+                ['Metadata Space Total', '10 MB'],
+                ['Metadata Space Used', '1 MB'],
+            ]
+        }
+
+    def mock_get_info_invalid_values(self):
+        return {
+            'DriverStatus': [
+                ['Metadata Space Available', '9 MB'],
+                ['Metadata Space Total', '10 MB'],
+                ['Metadata Space Used', '11 MB'],
+            ]
+        }
+
+    @mock.patch('docker.Client.info')
+    def test_devicemapper_disk_metrics(self, mock_info):
+        mock_info.return_value = self.mock_normal_get_info()
+
+        self.run_check(MOCK_CONFIG, force_reload=True)
+        self.assertMetric('docker.data.free', value=9e9)
+        self.assertMetric('docker.data.used', value=1e9)
+        self.assertMetric('docker.data.total', value=10e9)
+        self.assertMetric('docker.data.percent', value=10.0)
+        self.assertMetric('docker.metadata.free', value=9e6)
+        self.assertMetric('docker.metadata.used', value=1e6)
+        self.assertMetric('docker.metadata.total', value=10e6)
+        self.assertMetric('docker.metadata.percent', value=10.0)
+
+    @mock.patch('docker.Client.info')
+    def test_devicemapper_no_used_info(self, mock_info):
+        """Disk metrics collection should still work and `percent` can be calculated"""
+        mock_info.return_value = self.mock_get_info_no_used()
+
+        self.run_check(MOCK_CONFIG, force_reload=True)
+        self.assertMetric('docker.data.free', value=9e9)
+        self.assertMetric('docker.data.total', value=10e9)
+        self.assertMetric('docker.data.percent', value=10.0)
+        self.assertMetric('docker.metadata.free', value=9e6)
+        self.assertMetric('docker.metadata.total', value=10e6)
+        self.assertMetric('docker.metadata.percent', value=10.0)
+
+    @mock.patch('docker.Client.info')
+    def test_devicemapper_no_data_info(self, mock_info):
+        """Disk metrics collection should still partially work for metadata"""
+        mock_info.return_value = self.mock_get_info_no_data()
+
+        self.run_check(MOCK_CONFIG, force_reload=True)
+        self.assertMetric('docker.metadata.free', value=9e6)
+        self.assertMetric('docker.metadata.total', value=10e6)
+        self.assertMetric('docker.metadata.percent', value=10.0)
+
+    @mock.patch('docker.Client.info')
+    def test_devicemapper_invalid_values(self, mock_info):
+        """Invalid values are detected in _calc_percent_disk_stats, so percent should be missing"""
+        mock_info.return_value = self.mock_get_info_invalid_values()
+
+        self.run_check(MOCK_CONFIG, force_reload=True)
+        metric_names = [metric[0] for metric in self.metrics]
+        self.assertMetric('docker.metadata.free', value=9e6)
+        self.assertMetric('docker.metadata.used', value=11e6)
+        self.assertMetric('docker.metadata.total', value=10e6)
+        self.assertNotIn('docker.metadata.percent', metric_names)