[redis-sentinel] adding redis-sentinel integration with SDK scaffoldi…

…ng/testing facilities.

.travis.yml

@@ -58,6 +58,9 @@ install:
 script:
   - bundle exec rake prep_travis_ci
   - bundle exec rake ci:run
+  - bundle exec rake lint
+  - rake ci:run[default]
+  - rake ci:run[redis_sentinel]
   - bundle exec rake requirements
 
 # we should clean generated files before we save the cache

ci/redis_sentinel.rake

@@ -0,0 +1,69 @@
+require 'ci/common'
+
+def redis_sentinel_version
+  ENV['FLAVOR_VERSION'] || '2.4.12'
+end
+
+def redis_sentinel_rootdir
+  "#{ENV['INTEGRATIONS_DIR']}/redis_sentinel_#{redis_sentinel_version}"
+end
+
+namespace :ci do
+  namespace :redis_sentinel do |flavor|
+    task before_install: ['ci:common:before_install']
+
+    task install: ['ci:common:install'] do
+      use_venv = in_venv
+      install_requirements('redis_sentinel/requirements.txt',
+                           "--cache-dir #{ENV['PIP_CACHE']}",
+                           "#{ENV['VOLATILE_DIR']}/ci.log", use_venv)
+      # sample docker usage
+      # sh %(docker create -p XXX:YYY --name redis_sentinel source/redis_sentinel)
+      # sh %(docker start redis_sentinel)
+    end
+
+    task before_script: ['ci:common:before_script']
+
+    task :script, [:mocked] => ['ci:common:script'] do |_, attr|
+      ci_home = File.dirname(__FILE__)
+      mocked = attr[:mocked] || false
+      this_provides = [
+        'redis_sentinel'
+      ]
+      Rake::Task['ci:common:run_tests'].invoke(this_provides, ci_home, mocked)
+    end
+
+    task before_cache: ['ci:common:before_cache']
+
+    task cleanup: ['ci:common:cleanup']
+    # sample cleanup task
+    # task cleanup: ['ci:common:cleanup'] do 
+    #   sh %(docker stop redis_sentinel)
+    #   sh %(docker rm redis_sentinel)
+    # end
+
+    task :execute, :mocked do |_, attr|
+      mocked = attr[:mocked] || false
+      exception = nil
+      begin
+        if not mocked
+          %w(before_install install before_script).each do |u|
+            Rake::Task["#{flavor.scope.path}:#{u}"].invoke
+          end
+        end
+        Rake::Task["#{flavor.scope.path}:script"].invoke(mocked)
+        Rake::Task["#{flavor.scope.path}:before_cache"].invoke
+      rescue => e
+        exception = e
+        puts "Failed task: #{e.class} #{e.message}".red
+      end
+      if ENV['SKIP_CLEANUP']
+        puts 'Skipping cleanup, disposable environments are great'.yellow
+      else
+        puts 'Cleaning up'
+        Rake::Task["#{flavor.scope.path}:cleanup"].invoke
+      end
+      raise exception if exception
+    end
+  end
+end

circle.yml

@@ -38,6 +38,7 @@ test:
         - bundle exec rake prep_travis_ci
         - rake ci:run[default]
         - rake ci:run[snmpwalk]
+        - rake ci:run[redis_sentinel]
         - bundle exec rake requirements
     post:
         - if [[ $(docker ps -a -q) ]]; then docker stop $(docker ps -a -q); fi

redis_sentinel/README.md

@@ -0,0 +1,32 @@
+# Redis Sentinel
+
+## Overview
+
+Get metrics from Redis's Sentinel service in real time to:
+
+* Visualize and monitor sentinels states
+* Be notified about failovers
+
+## Installation
+
+Install the `dd-check-redis-sentinel` package manually or with your favorite configuration manager
+
+## Configuration
+
+Edit the `redis_sentinel.yaml` file to point to your server and port, set the masters to monitor
+
+## Validation
+
+When you run `datadog-agent info` you should see something like the following:
+
+    Checks
+    ======
+
+        redis_sentinel
+        --------------
+          - instance #0 [OK]
+          - Collected 39 metrics, 0 events & 7 service checks
+
+## Compatibility
+
+The Redis Sentinel check is compatible with all major platforms

redis_sentinel/check.py

@@ -0,0 +1,212 @@
+import time
+from collections import defaultdict
+
+# 3rd party
+import redis
+
+# project
+from checks import AgentCheck
+
+EVENT_TYPE = SOURCE_TYPE_NAME = 'redis_sentinel'
+
+
+class RedisSentinelCheck(AgentCheck):
+
+    def __init__(self, name, init_config, agentConfig, instances=None):
+        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
+        self._masters = defaultdict(lambda: "")
+
+    def check(self, instance):
+        redis_conn = redis.StrictRedis(
+            host=instance['sentinel_host'], port=instance['sentinel_port'], db=0
+        )
+        for master_name in instance['masters']:
+            base_tags = ['redis_name:%s' % master_name] + instance.get('tags', [])
+            try:
+                self._process_instance_master(redis_conn, master_name, base_tags)
+            except Exception as e:
+                self.warning("Error collecting metrics for master %s: %s" % (master_name, e))
+
+    def _process_instance_master(self, redis_conn, master_name, base_tags):
+        master_tags = self._process_master_stats(redis_conn, master_name, base_tags)
+        self._process_slaves_stats(redis_conn, master_name, base_tags, master_tags)
+        self._process_sentinels_stats(redis_conn, master_name, base_tags, master_tags)
+
+    def _process_sentinels_stats(self, redis_conn, master_name, base_tags, master_tags):
+        """
+            [{
+                'down-after-milliseconds': 5000,
+                'flags': 's_down,sentinel',
+                'ip': '10.1.2.3',
+                'is_disconnected': False,
+                'is_master': False,
+                'is_master_down': False,
+                'is_odown': False,
+                'is_sdown': True,
+                'is_sentinel': True,
+                'is_slave': False,
+                'last-hello-message': 12345678,
+                'last-ok-ping-reply': 12345678,
+                'last-ping-reply': 12345679
+                'last-ping-sent': 12345678,
+                'name': '10.1.2.3:26379',
+                'pending-commands': 78,
+                'port': 26379,
+                'runid': '123456789abcdef',
+                's-down-time': 12345678,
+                'voted-leader': '?',
+                'voted-leader-epoch': 0,
+            }]
+        """
+        sentinels_stats = redis_conn.sentinel_sentinels(master_name)
+        # sentinel_stats returns stats for other sentinels only
+        # so increment once for current sentinel
+        self.increment('redis.sentinel.ok_sentinels', tags=master_tags)
+        for stats in sentinels_stats:
+            sentinel_tags = ['sentinel_ip:%s' % stats['ip']] + base_tags
+            if stats['is_odown'] or stats['is_sdown']:  # sentinel keeps track of old sentinels
+                continue
+            self.increment('redis.sentinel.ok_sentinels', tags=master_tags)
+            self.gauge(
+                'redis.sentinel.pending_commands',
+                stats['pending-commands'], tags=['sentinel'] + sentinel_tags
+            )
+            self.gauge(
+                'redis.sentinel.ping_latency',
+                stats['last-ping-reply'] - stats['last-ping-sent'],
+                sentinel_tags
+            )
+            self.gauge(
+                'redis.sentinel.last_ok_ping_latency',
+                stats['last-ping-reply'] - stats['last-ok-ping-reply'],
+                sentinel_tags
+            )
+
+    def _process_slaves_stats(self, redis_conn, master_name, base_tags, master_tags):
+        """
+            [{
+                'down-after-milliseconds': 5000,
+                'flags': 'slave',
+                'info-refresh': 2628,
+                'ip': '10.1.2.3',
+                'is_disconnected': False,
+                'is_master': False,
+                'is_master_down': False,
+                'is_odown': False,
+                'is_sdown': False,
+                'is_sentinel': False,
+                'is_slave': True
+                'last-ok-ping-reply': 429,
+                'last-ping-reply': 429,
+                'last-ping-sent': 0,
+                'master-host': '10.1.2.3',
+                'master-link-down-time': 0,
+                'master-link-status': 'ok',
+                'master-port': 6379,
+                'name': '10.1.2.3:6379',
+                'pending-commands': 0,
+                'port': 6379,
+                'role-reported': 'slave',
+                'role-reported-time': 3124725,
+                'runid': '123456789abcdef',
+                'slave-priority': 100,
+                'slave-repl-offset': 12345678,
+            }]
+        """
+        slaves_stats = redis_conn.sentinel_slaves(master_name)
+        for stats in slaves_stats:
+            if stats['is_odown'] or stats['is_sdown']:  # sentinel keeps track of old slaves
+                continue
+            self.increment('redis.sentinel.ok_slaves', tags=master_tags)
+            slave_tags = ['slave_ip:%s' % stats['ip']] + base_tags
+            self.gauge(
+                'redis.sentinel.pending_commands', stats['pending-commands'],
+                tags=['slave'] + slave_tags
+            )
+
+            self.service_check(
+                'redis.sentinel.slave_is_disconnected',
+                AgentCheck.CRITICAL if stats['is_disconnected'] else AgentCheck.OK,
+                tags=slave_tags
+            )
+            self.service_check(
+                'redis.sentinel.slave_master_link_down',
+                AgentCheck.CRITICAL if stats['master-link-status'] != 'ok' else AgentCheck.OK,
+                tags=slave_tags
+            )
+
+    def _process_master_stats(self, redis_conn, master_name, base_tags):
+        """
+        {
+            'config-epoch': 94,
+            'down-after-milliseconds': 5000,
+            'failover-timeout': 60000,
+            'flags': 'master',
+            'info-refresh': 1234,
+            'ip': '10.1.2.3',
+            'is_disconnected': False,
+            'is_master': True,
+            'is_master_down': False,
+            'is_odown': False,
+            'is_sdown': False,
+            'is_sentinel': False,
+            'is_slave': False,
+            'last-ok-ping-reply': 49,
+            'last-ping-reply': 49,
+            'last-ping-sent': 0,
+            'name': 'delancie-backend',
+            'num-other-sentinels': 4
+            'num-slaves': 3,
+            'parallel-syncs': 10,
+            'pending-commands': 0,
+            'port': 6379,
+            'quorum': 2,
+            'role-reported': 'master',
+            'role-reported-time': 12345678,
+            'runid': '123456789abcdef',
+        }
+        """
+        stats = redis_conn.sentinel_master(master_name)
+        master_tags = ['master_ip:%s' % stats['ip']] + base_tags
+        self.gauge(
+            'redis.sentinel.pending_commands', stats['pending-commands'],
+            tags=['master'] + master_tags
+        )
+        self.gauge(
+            'redis.sentinel.known_slaves', stats['num-slaves'], tags=master_tags
+        )
+        self.gauge(
+            'redis.sentinel.known_sentinels',
+            stats['num-other-sentinels'] + 1,
+            tags=master_tags
+        )
+
+        self.service_check(
+            'redis.sentinel.master_is_disconnected',
+            AgentCheck.CRITICAL if stats['is_disconnected'] else AgentCheck.OK,
+            tags=master_tags
+        )
+        self.service_check(
+            'redis.sentinel.master_is_down',
+            AgentCheck.CRITICAL if stats['is_master_down'] else AgentCheck.OK,
+            tags=master_tags
+        )
+
+        if self._masters[master_name] != stats['ip']:
+            if self._masters[master_name] != "":  # avoid check initialization
+                self.increment('redis.sentinel.failover', tags=base_tags)
+                self.event({
+                    'timestamp': int(time.time()),
+                    'event_type': EVENT_TYPE,
+                    'msg_title': '%s failover from %s to %s' % (
+                        master_name, self._masters[master_name], stats['ip']
+                    ),
+                    'alert_type': 'info',
+                    "source_type_name": SOURCE_TYPE_NAME,
+                    "event_object": master_name,
+                    "tags": base_tags
+                })
+
+            self._masters[master_name] = stats['ip']
+
+        return master_tags

redis_sentinel/conf.yaml.example

@@ -0,0 +1,9 @@
+init_config:
+
+instances:
+  - sentinel_host: localhost
+    sentinel_port: 26379
+    tags: ['custom:tag']
+    masters:
+      - mymaster
+      - myothermaster

redis_sentinel/manifest.json

@@ -0,0 +1,15 @@
+{
+  "maintainer": "[email protected]",
+  "manifest_version": "0.1.0",
+  "max_agent_version": "6.0.0",
+  "min_agent_version": "5.6.3",
+  "name": "redis_sentinel",
+  "parameters": {
+    "creates_events": false,
+    "metrics_to_check": ["redis.sentinel.known_sentinels"],
+    "user_configured": false
+  },
+  "short_description": "Redis Sentinel provides high availability for Redis.",
+  "support": "contrib",
+  "version": "0.1.0"
+}

redis_sentinel/metadata.csv

@@ -0,0 +1,9 @@
+metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name
+redis.sentinel.known_sentinels,gauge,,instance,,number of sentinels detected,,redis_sentinel,known sentinels
+redis.sentinel.known_slaves,gauge,,instance,,number of slaves detected,,redis_sentinel,known slaves
+redis.sentinel.last_ok_ping_latency,gauge,,second,,number of seconds since last OK ping,,redis_sentinel,last ok latency
+redis.sentinel.ok_sentinels,gauge,,instance,,number of sentinels up and running,,redis_sentinel,ok sentinels
+redis.sentinel.ok_slaves,gauge,,instance,,number of slaves up and running,,redis_sentinel,ok slaves
+redis.sentinel.pending_commands,gauge,,command,,number of pending sentinel commands,,redis_sentinel,pending commands
+redis.sentinel.ping_latency,gauge,,millisecond,,latency of a sentinel ping,,redis_sentinel,ping latency
+redis.sentinel.failover,count,,occurrence,,number of failovers detected,,redis_sentinel,failovers

redis_sentinel/requirements.txt

@@ -0,0 +1 @@
+redis