From ac14c238fec406941b397a37113c825cea4eb007 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Fri, 28 Sep 2018 17:04:06 +0200 Subject: [PATCH 1/6] support the empty_default_hostname instance field This makes jmxfetch submit metrics and service-checks with the dummy `host:` tag. It is supported for metrics since agent5, and for service-checks in 6.6 --- .../java/org/datadog/jmxfetch/Instance.java | 21 ++++++-- .../org/datadog/jmxfetch/JMXAttribute.java | 8 +++- .../datadog/jmxfetch/JMXComplexAttribute.java | 4 +- .../datadog/jmxfetch/JMXSimpleAttribute.java | 5 +- .../datadog/jmxfetch/JMXTabularAttribute.java | 4 +- .../org/datadog/jmxfetch/TestInstance.java | 48 +++++++++++++++++-- .../resources/jmx_empty_default_hostname.yaml | 27 +++++++++++ 7 files changed, 100 insertions(+), 17 deletions(-) create mode 100644 src/test/resources/jmx_empty_default_hostname.yaml diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index 9b2f377a2..4815fcf41 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -57,6 +57,7 @@ public class Instance { private Connection connection; private AppConfig appConfig; private Boolean cassandraAliasing; + private Boolean emptyDefaultHostname; public Instance(Instance instance, AppConfig appConfig) { @@ -93,10 +94,16 @@ public Instance(LinkedHashMap instanceMap, LinkedHashMap> getMetrics() throws IOException { } return metrics; } - + public boolean timeToCollect() { if (this.minCollectionPeriod == null) { return true; @@ -349,13 +356,13 @@ private void getMatchingAttributes() { String attributeType = attributeInfo.getType(); if (SIMPLE_TYPES.contains(attributeType)) { LOGGER.debug(ATTRIBUTE + beanName + " : " + attributeInfo + " has attributeInfo simple type"); - jmxAttribute = new JMXSimpleAttribute(attributeInfo, beanName, instanceName, connection, tags, cassandraAliasing); + jmxAttribute = new JMXSimpleAttribute(attributeInfo, beanName, instanceName, connection, tags, cassandraAliasing, emptyDefaultHostname); } else if (COMPOSED_TYPES.contains(attributeType)) { LOGGER.debug(ATTRIBUTE + beanName + " : " + attributeInfo + " has attributeInfo composite type"); - jmxAttribute = new JMXComplexAttribute(attributeInfo, beanName, instanceName, connection, tags); + jmxAttribute = new JMXComplexAttribute(attributeInfo, beanName, instanceName, connection, tags, emptyDefaultHostname); } else if (MULTI_TYPES.contains(attributeType)) { LOGGER.debug(ATTRIBUTE + beanName + " : " + attributeInfo + " has attributeInfo tabular type"); - jmxAttribute = new JMXTabularAttribute(attributeInfo, beanName, instanceName, connection, tags); + jmxAttribute = new JMXTabularAttribute(attributeInfo, beanName, instanceName, connection, tags, emptyDefaultHostname); } else { try { LOGGER.debug(ATTRIBUTE + beanName + " : " + attributeInfo + " has an unsupported type: " + attributeType); @@ -446,6 +453,10 @@ public String[] getServiceCheckTags() { } } tags.add("instance:" + this.instanceName); + + if (this.emptyDefaultHostname) { + tags.add("host:"); + } return tags.toArray(new String[tags.size()]); } diff --git a/src/main/java/org/datadog/jmxfetch/JMXAttribute.java b/src/main/java/org/datadog/jmxfetch/JMXAttribute.java index d6ae20afa..6f44f4096 100644 --- a/src/main/java/org/datadog/jmxfetch/JMXAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JMXAttribute.java @@ -51,7 +51,8 @@ public abstract class JMXAttribute { private Boolean cassandraAliasing; JMXAttribute(MBeanAttributeInfo attribute, ObjectName beanName, String instanceName, - Connection connection, HashMap instanceTags, Boolean cassandraAliasing) { + Connection connection, HashMap instanceTags, Boolean cassandraAliasing, + Boolean emptyDefaultHostname) { this.attribute = attribute; this.beanName = beanName; this.matchingConf = null; @@ -73,6 +74,9 @@ public abstract class JMXAttribute { this.beanParameters = beanParametersHash; this.defaultTagsList = sanitizeParameters(beanParametersList); + if (emptyDefaultHostname) { + this.defaultTagsList.add("host:"); + } } /** @@ -303,7 +307,7 @@ private boolean matchBeanRegex(Filter filter, boolean matchIfNoRegex) { Matcher m = beanRegex.matcher(beanStringName); if(m.matches()) { - for (int i = 0; i<= m.groupCount(); i++) { + for (int i = 0; i<= m.groupCount(); i++) { this.beanParameters.put(Integer.toString(i), m.group(i)); } return true; diff --git a/src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java b/src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java index d38bf6218..350bde3f6 100644 --- a/src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java @@ -22,8 +22,8 @@ public class JMXComplexAttribute extends JMXAttribute { private HashMap> subAttributeList; public JMXComplexAttribute(MBeanAttributeInfo attribute, ObjectName beanName, String instanceName, - Connection connection, HashMap instanceTags) { - super(attribute, beanName, instanceName, connection, instanceTags, false); + Connection connection, HashMap instanceTags, Boolean emptyDefaultHostname) { + super(attribute, beanName, instanceName, connection, instanceTags, false, emptyDefaultHostname); this.subAttributeList = new HashMap>(); } diff --git a/src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java b/src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java index 195c53850..e3399b17d 100644 --- a/src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java @@ -19,8 +19,9 @@ public class JMXSimpleAttribute extends JMXAttribute { private String metricType; public JMXSimpleAttribute(MBeanAttributeInfo attribute, ObjectName beanName, String instanceName, - Connection connection, HashMap instanceTags, Boolean cassandraAliasing) { - super(attribute, beanName, instanceName, connection, instanceTags, cassandraAliasing); + Connection connection, HashMap instanceTags, Boolean cassandraAliasing, + Boolean emptyDefaultHostname) { + super(attribute, beanName, instanceName, connection, instanceTags, cassandraAliasing, emptyDefaultHostname); } @Override diff --git a/src/main/java/org/datadog/jmxfetch/JMXTabularAttribute.java b/src/main/java/org/datadog/jmxfetch/JMXTabularAttribute.java index e4e121ac1..ecaa61252 100644 --- a/src/main/java/org/datadog/jmxfetch/JMXTabularAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JMXTabularAttribute.java @@ -27,8 +27,8 @@ public class JMXTabularAttribute extends JMXAttribute { private HashMap>> subAttributeList; public JMXTabularAttribute(MBeanAttributeInfo attribute, ObjectName beanName, String instanceName, - Connection connection, HashMap instanceTags) { - super(attribute, beanName, instanceName, connection, instanceTags, false); + Connection connection, HashMap instanceTags, Boolean emptyDefaultHostname) { + super(attribute, beanName, instanceName, connection, instanceTags, false, emptyDefaultHostname); subAttributeList = new HashMap>>(); } diff --git a/src/test/java/org/datadog/jmxfetch/TestInstance.java b/src/test/java/org/datadog/jmxfetch/TestInstance.java index 2e5e6c763..80a7936e1 100644 --- a/src/test/java/org/datadog/jmxfetch/TestInstance.java +++ b/src/test/java/org/datadog/jmxfetch/TestInstance.java @@ -1,29 +1,33 @@ package org.datadog.jmxfetch; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; import java.util.HashMap; import java.util.LinkedList; +import java.util.Arrays; +import java.util.List; import org.apache.log4j.Logger; import org.junit.Test; public class TestInstance extends TestCommon { private final static Logger LOGGER = Logger.getLogger("Test Instance"); - + @Test public void testMinCollectionInterval() throws Exception { registerMBean(new SimpleTestJavaApp(), "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); initApplication("jmx_min_collection_period.yml"); - + run(); LinkedList> metrics = getMetrics(); assertEquals(11, metrics.size()); - run(); metrics = getMetrics(); assertEquals(0, metrics.size()); - + LOGGER.info("sleeping before the next collection"); Thread.sleep(5000); run(); @@ -31,4 +35,40 @@ public void testMinCollectionInterval() throws Exception { assertEquals(11, metrics.size()); } + // assertHostnameTags is used by testEmptyDefaultHostname + private void assertHostnameTags(List tagList) throws Exception { + // Fixed instance tag + assertTrue(tagList.contains(new String("jmx:fetch"))); + + if (tagList.contains(new String("instance:jmx_test_default_hostname"))) { + // Nominal case + assertFalse(tagList.contains(new String("host:"))); + } else if (tagList.contains(new String("instance:jmx_test_no_hostname"))) { + // empty_default_hostname case + assertTrue(tagList.contains(new String("host:"))); + } else { + fail("unexpected instance tag"); + } + } + + @Test + public void testEmptyDefaultHostname() throws Exception { + registerMBean(new SimpleTestJavaApp(), "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); + initApplication("jmx_empty_default_hostname.yaml"); + run(); + + LinkedList> metrics = getMetrics(); + assertEquals(20, metrics.size()); + for (HashMap metric : metrics) { + String[] tags = (String[]) metric.get("tags"); + this.assertHostnameTags(Arrays.asList(tags)); + } + + LinkedList> serviceChecks = getServiceChecks(); + assertEquals(2, serviceChecks.size()); + for (HashMap sc : serviceChecks) { + String[] tags = (String[]) sc.get("tags"); + this.assertHostnameTags(Arrays.asList(tags)); + } + } } diff --git a/src/test/resources/jmx_empty_default_hostname.yaml b/src/test/resources/jmx_empty_default_hostname.yaml new file mode 100644 index 000000000..464160882 --- /dev/null +++ b/src/test/resources/jmx_empty_default_hostname.yaml @@ -0,0 +1,27 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_test_default_hostname + tags: + - jmx:fetch + conf: + - include: + domain: org.datadog.jmxfetch.test + attribute: + ShouldBe100: + metric_type: gauge + alias: this.is.100.$foo.$qux + + - process_name_regex: .*surefire.* + empty_default_hostname: true + name: jmx_test_no_hostname + tags: + - jmx:fetch + conf: + - include: + domain: org.datadog.jmxfetch.test + attribute: + ShouldBe100: + metric_type: gauge + alias: this.is.100.$foo.$qux From c8ccc035872f66cfa8cda11fb8a30ad47752f560 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 2 Oct 2018 13:33:35 +0200 Subject: [PATCH 2/6] fix ident --- src/test/java/org/datadog/jmxfetch/TestInstance.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/datadog/jmxfetch/TestInstance.java b/src/test/java/org/datadog/jmxfetch/TestInstance.java index 80a7936e1..ea9454a24 100644 --- a/src/test/java/org/datadog/jmxfetch/TestInstance.java +++ b/src/test/java/org/datadog/jmxfetch/TestInstance.java @@ -62,12 +62,12 @@ public void testEmptyDefaultHostname() throws Exception { for (HashMap metric : metrics) { String[] tags = (String[]) metric.get("tags"); this.assertHostnameTags(Arrays.asList(tags)); - } + } LinkedList> serviceChecks = getServiceChecks(); assertEquals(2, serviceChecks.size()); for (HashMap sc : serviceChecks) { - String[] tags = (String[]) sc.get("tags"); + String[] tags = (String[]) sc.get("tags"); this.assertHostnameTags(Arrays.asList(tags)); } } From 6d1dce4fd597199261fda3800a4d7e7bc5a7e049 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Wed, 10 Oct 2018 10:39:06 +0200 Subject: [PATCH 3/6] use base boolean, catch NullPointerException --- src/main/java/org/datadog/jmxfetch/Instance.java | 9 +++++---- src/main/java/org/datadog/jmxfetch/JMXAttribute.java | 2 +- .../java/org/datadog/jmxfetch/JMXComplexAttribute.java | 2 +- .../java/org/datadog/jmxfetch/JMXSimpleAttribute.java | 2 +- .../java/org/datadog/jmxfetch/JMXTabularAttribute.java | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index 4815fcf41..15203e14d 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -57,7 +57,7 @@ public class Instance { private Connection connection; private AppConfig appConfig; private Boolean cassandraAliasing; - private Boolean emptyDefaultHostname; + private boolean emptyDefaultHostname; public Instance(Instance instance, AppConfig appConfig) { @@ -94,13 +94,14 @@ public Instance(LinkedHashMap instanceMap, LinkedHashMap instanceTags, Boolean cassandraAliasing, - Boolean emptyDefaultHostname) { + boolean emptyDefaultHostname) { this.attribute = attribute; this.beanName = beanName; this.matchingConf = null; diff --git a/src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java b/src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java index 350bde3f6..a67042474 100644 --- a/src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JMXComplexAttribute.java @@ -22,7 +22,7 @@ public class JMXComplexAttribute extends JMXAttribute { private HashMap> subAttributeList; public JMXComplexAttribute(MBeanAttributeInfo attribute, ObjectName beanName, String instanceName, - Connection connection, HashMap instanceTags, Boolean emptyDefaultHostname) { + Connection connection, HashMap instanceTags, boolean emptyDefaultHostname) { super(attribute, beanName, instanceName, connection, instanceTags, false, emptyDefaultHostname); this.subAttributeList = new HashMap>(); } diff --git a/src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java b/src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java index e3399b17d..10932b36c 100644 --- a/src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JMXSimpleAttribute.java @@ -19,7 +19,7 @@ public class JMXSimpleAttribute extends JMXAttribute { private String metricType; public JMXSimpleAttribute(MBeanAttributeInfo attribute, ObjectName beanName, String instanceName, - Connection connection, HashMap instanceTags, Boolean cassandraAliasing, + Connection connection, HashMap instanceTags, boolean cassandraAliasing, Boolean emptyDefaultHostname) { super(attribute, beanName, instanceName, connection, instanceTags, cassandraAliasing, emptyDefaultHostname); } diff --git a/src/main/java/org/datadog/jmxfetch/JMXTabularAttribute.java b/src/main/java/org/datadog/jmxfetch/JMXTabularAttribute.java index ecaa61252..441b063a3 100644 --- a/src/main/java/org/datadog/jmxfetch/JMXTabularAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JMXTabularAttribute.java @@ -27,7 +27,7 @@ public class JMXTabularAttribute extends JMXAttribute { private HashMap>> subAttributeList; public JMXTabularAttribute(MBeanAttributeInfo attribute, ObjectName beanName, String instanceName, - Connection connection, HashMap instanceTags, Boolean emptyDefaultHostname) { + Connection connection, HashMap instanceTags, boolean emptyDefaultHostname) { super(attribute, beanName, instanceName, connection, instanceTags, false, emptyDefaultHostname); subAttributeList = new HashMap>>(); } From 6d874b6a517ff0019038d3cf6be4c319ca4608be Mon Sep 17 00:00:00 2001 From: Jaime Fullaondo Date: Fri, 30 Nov 2018 00:59:44 +0100 Subject: [PATCH 4/6] [test] we've got more metrics now - bumping metric count --- src/test/java/org/datadog/jmxfetch/TestInstance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/datadog/jmxfetch/TestInstance.java b/src/test/java/org/datadog/jmxfetch/TestInstance.java index b96464a4f..0fe3dd244 100644 --- a/src/test/java/org/datadog/jmxfetch/TestInstance.java +++ b/src/test/java/org/datadog/jmxfetch/TestInstance.java @@ -59,7 +59,7 @@ public void testEmptyDefaultHostname() throws Exception { run(); LinkedList> metrics = getMetrics(); - assertEquals(20, metrics.size()); + assertEquals(28, metrics.size()); for (HashMap metric : metrics) { String[] tags = (String[]) metric.get("tags"); this.assertHostnameTags(Arrays.asList(tags)); From 91a79401e8d450ac6ef61548d4e23313d8cc3685 Mon Sep 17 00:00:00 2001 From: Jaime Fullaondo Date: Fri, 30 Nov 2018 01:32:45 +0100 Subject: [PATCH 5/6] [test][travis] logs are too verbose, lets curb it --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fc5611fc1..3d6ecccd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ jdk: - openjdk7 install: mvn install -Dhttps.protocols=TLSv1.2 -DskipTests=true -Dmaven.javadoc.skip=true -B -V -script: mvn test -B -Dhttps.protocols=TLSv1.2 +script: mvn test -B -Dhttps.protocols=TLSv1.2 -Dlog4j.configuration=log4j.test.properties addons: hostname: dd-jmxfetch-testhost From 1bb7f1758ed00531f28781167a348e50cbd07be3 Mon Sep 17 00:00:00 2001 From: Jaime Fullaondo Date: Fri, 30 Nov 2018 01:34:58 +0100 Subject: [PATCH 6/6] [test][travis] make it explicit its for travis + include actual file --- .travis.yml | 2 +- log4j.travis.properties | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 log4j.travis.properties diff --git a/.travis.yml b/.travis.yml index 3d6ecccd8..817f20ee1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ jdk: - openjdk7 install: mvn install -Dhttps.protocols=TLSv1.2 -DskipTests=true -Dmaven.javadoc.skip=true -B -V -script: mvn test -B -Dhttps.protocols=TLSv1.2 -Dlog4j.configuration=log4j.test.properties +script: mvn test -B -Dhttps.protocols=TLSv1.2 -Dlog4j.configuration=log4j.travis.properties addons: hostname: dd-jmxfetch-testhost diff --git a/log4j.travis.properties b/log4j.travis.properties new file mode 100644 index 000000000..0641702ac --- /dev/null +++ b/log4j.travis.properties @@ -0,0 +1,8 @@ +log4j.rootLogger=INFO,stdout +log4j.logger.com.endeca=INFO +# Logger for crawl metrics +log4j.logger.com.endeca.itl.web.metrics=INFO + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%p\t%d{ISO8601}\t%r\t%c\t[%t]\t%m%n