Merge pull request #93 from ipfs/monitoring-fixes

Monitoring fixes

solarnet/metrics.yml

@@ -9,7 +9,9 @@
 
 - hosts: metrics
   vars:
-    gateway_group: gateway
+    node_targets: "{{ cjdns_identities.keys() }}"
+    gateway_targets: "{{ groups.gateway }}"
+    storage_targets: "{{ groups.storage }}"
   pre_tasks:
     - include_vars: secrets_plaintext/secrets.yml
   handlers:

solarnet/roles/docker/tasks/main.yml

@@ -23,7 +23,7 @@
 # XXX migration
 - lineinfile:
     dest: /etc/default/docker
-    line: 'DOCKER_OPTS=--log-driver=none'
+    line: 'DOCKER_OPTS="--log-driver=none"'
     state: absent
   notify:
     - restart docker
@@ -35,3 +35,9 @@
     state: present
   notify:
     - restart docker
+
+- name: docker service
+  service:
+    name: docker
+    enabled: yes
+    state: started

solarnet/roles/metrics/files/Dockerfile

@@ -24,6 +24,7 @@ WORKDIR    /prometheus
 ENTRYPOINT [ "/bin/prometheus" ]
 CMD        [ "-config.file=/etc/prometheus/prometheus.yml", \
              "-storage.local.path=/prometheus", \
+             "-storage.local.memory-chunks=512288", \
              "-web.path-prefix=/prometheus", \
              "-web.console.libraries=/etc/prometheus/console_libraries", \
              "-web.console.templates=/etc/prometheus/consoles" ]

solarnet/roles/metrics/templates/prometheus.yml.j2

@@ -1,27 +1,30 @@
 ---
 global:
   scrape_interval:     15s
+  scrape_timeout:      10s
   evaluation_interval: 15s
-  labels:
-    monitor: '{{ ansible_hostname }}'
 
 scrape_configs:
-  - job_name: 'ipfs'
-    scrape_interval: 5s
-    scrape_timeout: 10s
+  - job_name: 'gateway'
     metrics_path: '/debug/metrics/prometheus'
     target_groups:
       - targets:
-{% for hostname in groups[gateway_group] %}
+{% for hostname in gateway_targets %}
         - '[{{ cjdns_identities[hostname].ipv6 }}]:5001'
 {% endfor %}
 
-  - job_name: 'node'
-    scrape_interval: 5s
-    scrape_timeout: 10s
+  - job_name: 'storage'
+    metrics_path: '/debug/metrics/prometheus'
+    target_groups:
+      - targets:
+{% for hostname in storage_targets %}
+        - '[{{ cjdns_identities[hostname].ipv6 }}]:5001'
+{% endfor %}
+
+  - job_name: 'host'
     metrics_path: '/metrics'
     target_groups:
       - targets:
-{% for hostname in cjdns_identities.keys() %}
+{% for hostname in node_targets %}
         - '[{{ cjdns_identities[hostname].ipv6 }}]:9100'
 {% endfor %}

solarnet/roles/node_exporter/templates/upstart_node_exporter.conf.j2

@@ -6,4 +6,4 @@ stop on starting rc RUNLEVEL=[016]
 respawn
 respawn limit unlimited
 
-exec /usr/bin/node_exporter --web.listen-address="127.0.0.1:9100"
+exec /usr/bin/node_exporter --web.listen-address="127.0.0.1:9100" -collectors.enabled=filesystem,netdev