Seth/cassandra metrics update

.gitignore

@@ -7,6 +7,8 @@ target/*
 .classpath
 .project
 /target
+.idea
+*.iml
 
 *.ucls
 

src/main/java/org/datadog/jmxfetch/JMXAttribute.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
@@ -28,6 +29,7 @@ public abstract class JMXAttribute {
     private static final String ALL_CAP_PATTERN = "([a-z0-9])([A-Z])";
     private static final String METRIC_REPLACEMENT = "([^a-zA-Z0-9_.]+)|(^[^a-zA-Z]+)";
     private static final String DOT_UNDERSCORE = "_*\\._*";
+    protected static final String CASSANDRA_DOMAIN = "org.apache.cassandra.metrics";
     private MBeanAttributeInfo attribute;
     private Connection connection;
     private ObjectInstance jmxInstance;
@@ -55,29 +57,37 @@ public abstract class JMXAttribute {
         String[] splitBeanName = this.beanName.split(":");
         String domain = splitBeanName[0];
         String beanParameters = splitBeanName[1];
-        LinkedList<String> defaultTags = getBeanTags(instanceName, domain, beanParameters, instanceTags);
-        HashMap<String, String> beanParametersHash = getBeanParametersHash(defaultTags);
+        HashMap<String, String> beanParametersHash = getBeanParametersHash(beanParameters);
+        LinkedList<String> defaultTags = getBeanTags(instanceName, domain, beanParametersHash, instanceTags);
 
         this.domain = domain;
         this.beanParameters = beanParametersHash;
         this.defaultTagsList = defaultTags;
     }
 
-    private static HashMap<String, String> getBeanParametersHash(LinkedList<String> beanParameters) {
-        HashMap<String, String> beanParams = new HashMap<String, String>();
+    private static HashMap<String, String> getBeanParametersHash(String beanParametersString) {
+        String[] beanParameters = beanParametersString.split(",");
+        HashMap<String, String> beanParamsMap = new HashMap<String, String>(beanParameters.length);
         for (String param : beanParameters) {
-            String[] paramSplit = param.split(":");
-            beanParams.put(new String(paramSplit[0]), new String(paramSplit[1]));
+            String[] paramSplit = param.split("=");
+            beanParamsMap.put(new String(paramSplit[0]), new String(paramSplit[1]));
         }
 
-        return beanParams;
+        return beanParamsMap;
     }
 
 
-    private static LinkedList<String> getBeanTags(String instanceName, String domain, String beanParameters, HashMap<String, String> instanceTags) {
-        LinkedList<String> beanTags = new LinkedList<String>(Arrays.asList(new String(beanParameters).replace("=", ":").split(",")));
+    private static LinkedList<String> getBeanTags(String instanceName, String domain, Map<String, String> beanParameters, Map<String, String> instanceTags) {
+        LinkedList<String> beanTags = new LinkedList<String>();
         beanTags.add("instance:" + instanceName);
         beanTags.add("jmx_domain:" + domain);
+        if (domain.equals(CASSANDRA_DOMAIN)) {
+            beanTags.addAll(getCassandraBeanTags(beanParameters));
+        } else {
+            for (Map.Entry<String, String> param : beanParameters.entrySet()) {
+                beanTags.add(param.getKey() + ":" + param.getValue());
+            }
+        }
 
         if (instanceTags != null) {
             for (Map.Entry<String, String> tag : instanceTags.entrySet()) {
@@ -88,6 +98,22 @@ private static LinkedList<String> getBeanTags(String instanceName, String domain
         return beanTags;
     }
 
+    private static Collection<String> getCassandraBeanTags(Map<String, String> beanParameters) {
+        Collection<String> tags = new LinkedList<String>();
+        for (Map.Entry<String, String> param : beanParameters.entrySet()) {
+            if (param.getKey().equals("name")) {
+                //This is already in the alias
+                continue;
+            } else if (param.getKey().equals("scope")) {
+                String type = beanParameters.get("type");
+                tags.add(type + ":" + param.getValue());
+            } else {
+                tags.add(param.getKey() + ":" + param.getValue());
+            }
+        }
+        return tags;
+    }
+
     static String convertMetricName(String metricName) {
         metricName = metricName.replaceAll(FIRST_CAP_PATTERN, "$1_$2");
         metricName = metricName.replaceAll(ALL_CAP_PATTERN, "$1_$2").toLowerCase();
@@ -297,11 +323,15 @@ protected String[] getTags() {
         return tags;
     }
 
-    String getBeanName() {
-        return beanName;
-    }
-
     String getAttributeName() {
         return attributeName;
     }
+
+    protected String getDomain() {
+        return domain;
+    }
+
+    protected HashMap<String, String> getBeanParameters() {
+        return beanParameters;
+    }
 }

src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java

@@ -118,9 +118,9 @@ private String getAlias(String subAttribute) {
         if (include.getAttribute() instanceof LinkedHashMap<?, ?>) {
             return ((LinkedHashMap<String, LinkedHashMap<String, String>>) (include.getAttribute())).get(subAttributeName).get("alias");
         } else if (conf.get("metric_prefix") != null) {
-            return conf.get("metric_prefix") + "." + getBeanName().split(":")[0] + "." + subAttributeName;
+            return conf.get("metric_prefix") + "." + getDomain() + "." + subAttributeName;
         }
-        return "jmx." + getBeanName().split(":")[0] + "." + subAttributeName;
+        return "jmx." + getDomain() + "." + subAttributeName;
     }
 
 

src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java

@@ -5,6 +5,7 @@
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
+import java.util.Map;
 
 import javax.management.AttributeNotFoundException;
 import javax.management.InstanceNotFoundException;
@@ -90,14 +91,31 @@ private String getAlias() {
             LinkedHashMap<String, LinkedHashMap<String, String>> attribute = (LinkedHashMap<String, LinkedHashMap<String, String>>) (include.getAttribute());
             alias = attribute.get(getAttribute().getName()).get("alias");
         } else if (conf.get("metric_prefix") != null) {
-            alias = conf.get("metric_prefix") + "." + getBeanName().split(":")[0] + "." + getAttributeName();
+            alias = conf.get("metric_prefix") + "." + getDomain() + "." + getAttributeName();
+        } else if (getDomain().startsWith("org.apache.cassandra")) {
+            alias = getCassandraAlias();
         } else {
-            alias = "jmx." + getBeanName().split(":")[0] + "." + getAttributeName();
+            alias = "jmx." + getDomain() + "." + getAttributeName();
         }
         alias = convertMetricName(alias);
         return alias;
     }
 
+    private String getCassandraAlias() {
+        if (getDomain().equals(CASSANDRA_DOMAIN)) {
+            Map<String, String> beanParameters = getBeanParameters();
+            String type = beanParameters.get("type");
+            String metricName = beanParameters.get("name");
+            String attributeName = getAttributeName();
+            if (attributeName.equals("Value")) {
+                return "cassandra." + metricName;
+            }
+            return "cassandra." + metricName + "." + attributeName;
+        }
+        //Deprecated Cassandra metric.  Remove domain prefix.
+        return getDomain().replace("org.apache.", "") + "." + getAttributeName();
+    }
+
     private String getMetricType() {
         Filter include = getMatchingConf().getInclude();
         if (metricType != null) {

src/main/java/org/datadog/jmxfetch/reporter/Reporter.java

@@ -60,8 +60,6 @@ public void sendMetrics(LinkedList<HashMap<String, Object>> metrics, String inst
             // We need to edit metrics for legacy reasons (rename metrics, etc)
             HashMap<String, Object> metric = new HashMap<String, Object>(m);
 
-            postProcess(metric);
-
             Double currentValue = (Double) metric.get("value");
             if (currentValue.isNaN() || currentValue.isInfinite()) {
                 continue;
@@ -129,7 +127,7 @@ public int getServiceCheckCount(String checkName){
         Integer scCount = this.serviceCheckCount.get(checkName);
         return (scCount == null) ? 0 : scCount.intValue();
     }
-    
+
     public void resetServiceCheckCount(String checkName){
         this.serviceCheckCount.put(checkName, new Integer(0));
     }

src/test/java/org/datadog/jmxfetch/TestApp.java

@@ -83,6 +83,92 @@ public void testBeanTags() throws Exception {
         mbs.unregisterMBean(objectName);
     }
 
+    @Test
+    public void testCassandraBean() throws Exception {
+        // We expose a few metrics through JMX
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+        ObjectName objectName = new ObjectName("org.apache.cassandra.metrics:keyspace=MyKeySpace,type=ColumnFamily,scope=MyColumnFamily,name=PendingTasks");
+        SimpleTestJavaApp testApp = new SimpleTestJavaApp();
+        mbs.registerMBean(testApp, objectName);
+
+        // Initializing application
+        AppConfig appConfig = new AppConfig();
+        App app = initApp("jmx_cassandra.yaml", appConfig);
+
+        // Collecting metrics
+        app.doIteration();
+        LinkedList<HashMap<String, Object>> metrics = ((ConsoleReporter) appConfig.getReporter()).getMetrics();
+
+        // 14 = 13 metrics from java.lang + 1 metric explicitly defined in the yaml config file
+        assertEquals(14, metrics.size());
+
+
+        // Fetching our 'defined' metric tags
+        Boolean foundCassandraBean = false;
+        for (HashMap<String, Object> m : metrics) {
+            String name = (String) (m.get("name"));
+            if(!name.equals("cassandra.pending_tasks.should_be100")){
+                continue;
+            }
+            foundCassandraBean = true;
+            String[] tags = (String[]) (m.get("tags"));
+            Set<String> tagsSet = new HashSet<String>(Arrays.asList(tags));
+
+            // We should find bean parameters as tags
+            assertEquals(5, tags.length);
+            assertTrue(tagsSet.contains("type:ColumnFamily"));
+            assertTrue(tagsSet.contains("keyspace:MyKeySpace"));
+            assertTrue(tagsSet.contains("ColumnFamily:MyColumnFamily"));
+            assertTrue(tagsSet.contains("jmx_domain:org.apache.cassandra.metrics"));
+            assertTrue(tagsSet.contains("instance:jmx_test_instance"));
+        }
+        assertTrue(foundCassandraBean);
+        mbs.unregisterMBean(objectName);
+    }
+
+    @Test
+    public void testCassandraDeprecatedBean() throws Exception {
+        // We expose a few metrics through JMX
+        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+        ObjectName objectName = new ObjectName("org.apache.cassandra.db:type=ColumnFamilies,keyspace=MyKeySpace,columnfamily=MyColumnFamily");
+        SimpleTestJavaApp testApp = new SimpleTestJavaApp();
+        mbs.registerMBean(testApp, objectName);
+
+        // Initializing application
+        AppConfig appConfig = new AppConfig();
+        App app = initApp("jmx_cassandra_deprecated.yaml", appConfig);
+
+        // Collecting metrics
+        app.doIteration();
+        LinkedList<HashMap<String, Object>> metrics = ((ConsoleReporter) appConfig.getReporter()).getMetrics();
+
+        // 14 = 13 metrics from java.lang + 1 metric explicitly defined in the yaml config file
+        assertEquals(14, metrics.size());
+
+
+        // Fetching our 'defined' metric tags
+        Boolean foundCassandraBean = false;
+        for (HashMap<String, Object> m : metrics) {
+            String name = (String) (m.get("name"));
+            if(!name.equals("cassandra.db.should_be100")){
+                continue;
+            }
+            foundCassandraBean = true;
+            String[] tags = (String[]) (m.get("tags"));
+            Set<String> tagsSet = new HashSet<String>(Arrays.asList(tags));
+
+            // We should find bean parameters as tags
+            assertEquals(5, tags.length);
+            assertTrue(tagsSet.contains("type:ColumnFamilies"));
+            assertTrue(tagsSet.contains("keyspace:MyKeySpace"));
+            assertTrue(tagsSet.contains("columnfamily:MyColumnFamily"));
+            assertTrue(tagsSet.contains("jmx_domain:org.apache.cassandra.db"));
+            assertTrue(tagsSet.contains("instance:jmx_test_instance"));
+        }
+        assertTrue(foundCassandraBean);
+        mbs.unregisterMBean(objectName);
+    }
+
     @Test
     public void testDomainInclude() throws Exception {
         // We expose a few metrics through JMX

src/test/resources/jmx_cassandra.yaml

@@ -0,0 +1,10 @@
+init_config:
+
+instances:
+    -   process_name_regex: .*surefire.*
+        name: jmx_test_instance
+        conf:
+            - include:
+               bean: org.apache.cassandra.metrics:keyspace=MyKeySpace,type=ColumnFamily,scope=MyColumnFamily,name=PendingTasks
+               attribute:
+                    - ShouldBe100

src/test/resources/jmx_cassandra_deprecated.yaml

@@ -0,0 +1,10 @@
+init_config:
+
+instances:
+    -   process_name_regex: .*surefire.*
+        name: jmx_test_instance
+        conf:
+            - include:
+               bean: org.apache.cassandra.db:type=ColumnFamilies,keyspace=MyKeySpace,columnfamily=MyColumnFamily
+               attribute:
+                    - ShouldBe100