diff --git a/.travis.yml b/.travis.yml index 2fc849e24..047bf9955 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ jdk: - oraclejdk8 - oraclejdk7 - openjdk7 -- openjdk6 addons: hostname: dd-jmxfetch-testhost diff --git a/pom.xml b/pom.xml index 368fca8bd..b916d87be 100644 --- a/pom.xml +++ b/pom.xml @@ -12,6 +12,7 @@ http://maven.apache.org + 1.6 2.4 2.6 3.5 @@ -20,8 +21,8 @@ 1.35 4.11 1.2.17 - 1.4 - 2.2.27 + 2.9.0 + 2.2.27 2.9 UTF-8 1.13 @@ -65,11 +66,23 @@ com.datadoghq java-dogstatsd-client ${java-dogstatsd-client.version} - + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + - com.google.code.gson - gson - ${gson.version} + com.fasterxml.jackson.core + jackson-core + ${jackson.version} log4j diff --git a/src/main/java/org/datadog/jmxfetch/App.java b/src/main/java/org/datadog/jmxfetch/App.java index 26932e728..8635d51a6 100644 --- a/src/main/java/org/datadog/jmxfetch/App.java +++ b/src/main/java/org/datadog/jmxfetch/App.java @@ -29,6 +29,7 @@ import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.commons.lang3.CharEncoding; +import org.apache.commons.io.IOUtils; import org.datadog.jmxfetch.reporter.Reporter; import org.datadog.jmxfetch.util.CustomLogger; import org.datadog.jmxfetch.util.FileHelper; @@ -38,6 +39,8 @@ import com.beust.jcommander.ParameterException; import com.google.common.primitives.Bytes; +import com.fasterxml.jackson.core.JsonProcessingException; + @SuppressWarnings("unchecked") public class App { @@ -52,16 +55,24 @@ public class App { private static final int AD_MAX_MAG_INSTANCES = 4; // 1000 instances ought to be enough for anyone private static int loopCounter; - private AtomicBoolean reinit = new AtomicBoolean(false); + private int lastJSONConfigTS; + private HashMap adJSONConfigs; private ConcurrentHashMap configs; - private ConcurrentHashMap adConfigs = new ConcurrentHashMap(); + private ConcurrentHashMap adPipeConfigs = new ConcurrentHashMap(); private ArrayList instances = new ArrayList(); private LinkedList brokenInstances = new LinkedList(); + private AtomicBoolean reinit = new AtomicBoolean(false); + private AppConfig appConfig; + private HttpClient client; public App(AppConfig appConfig) { this.appConfig = appConfig; + // setup client + if (appConfig.remoteEnabled()) { + client = new HttpClient(appConfig.getIPCHost(), appConfig.getIPCPort(), false); + } this.configs = getConfigs(appConfig); } @@ -120,6 +131,9 @@ public static void main(String[] args) { LOGGER.info("JMX Fetch has started"); + //set up the config status + config.updateStatus(); + App app = new App(config); // Initiate JMX Connections, get attributes that match the yaml configuration @@ -243,7 +257,7 @@ void start() { long delta_s = 0; FileInputStream adPipe = null; - if(appConfig.getAutoDiscoveryEnabled()) { + if(appConfig.getAutoDiscoveryPipeEnabled()) { LOGGER.info("Auto Discovery enabled"); adPipe = newAutoDiscoveryPipe(); try { @@ -261,11 +275,11 @@ void start() { System.exit(0); } - // any SD configs waiting in pipe? - if(adPipe == null && appConfig.getAutoDiscoveryEnabled()) { + if(adPipe == null && appConfig.getAutoDiscoveryPipeEnabled()) { // If SD is enabled and the pipe is not open, retry opening pipe adPipe = newAutoDiscoveryPipe(); } + // any AutoDiscovery configs waiting? try { if(adPipe != null && adPipe.available() > 0) { byte[] buffer = new byte[0]; @@ -291,10 +305,14 @@ void start() { } setReinit(processAutoDiscovery(buffer)); } + + if(appConfig.remoteEnabled()) { + setReinit(getJSONConfigs()); + } } catch(IOException e) { LOGGER.warn("Unable to read from pipe - Service Discovery configuration may have been skipped."); } catch(Exception e) { - LOGGER.warn("Unknown problem parsing auto-discovery configuration: " + e); + LOGGER.warn("Problem parsing auto-discovery configuration: " + e); } long start = System.currentTimeMillis(); @@ -306,7 +324,7 @@ void start() { doIteration(); } else { LOGGER.warn("No instance could be initiated. Retrying initialization."); - appConfig.getStatus().flush(appConfig.getIPCPort()); + appConfig.getStatus().flush(); configs = getConfigs(appConfig); init(true); } @@ -425,7 +443,7 @@ public void doIteration() { } try { - appConfig.getStatus().flush(appConfig.getIPCPort()); + appConfig.getStatus().flush(); } catch (Exception e) { LOGGER.error("Unable to flush stats.", e); } @@ -460,12 +478,16 @@ public boolean addConfig(String name, YamlParser config) { return false; } - this.adConfigs.put(name, config); + this.adPipeConfigs.put(name, config); this.setReinit(true); return true; } + public boolean addJsonConfig(String name, String json) { + return false; + } + private ConcurrentHashMap getConfigs(AppConfig config) { ConcurrentHashMap configs = new ConcurrentHashMap(); YamlParser fileConfig; @@ -504,6 +526,47 @@ private ConcurrentHashMap getConfigs(AppConfig config) { return configs; } + private boolean getJSONConfigs() { + HttpClient.HttpResponse response; + boolean update = false; + + if (this.client == null) { + return update; + } + + try { + String uripath = "agent/jmx/configs?timestamp="+lastJSONConfigTS; + response = client.request("GET", "", uripath); + if (!response.isResponse2xx()) { + LOGGER.warn("Failed collecting JSON configs: [" + + response.getResponseCode() +"] " + + response.getResponseBody()); + return update; + } else if (response.getResponseCode() == 204) { + LOGGER.debug("No configuration changes..."); + return update; + } + + LOGGER.debug("Received the following JSON configs: " + response.getResponseBody()); + + InputStream jsonInputStream = IOUtils.toInputStream(response.getResponseBody(), "UTF-8"); + JsonParser parser = new JsonParser(jsonInputStream); + int timestamp = ((Integer) parser.getJsonTimestamp()).intValue(); + if (timestamp > lastJSONConfigTS) { + adJSONConfigs = (HashMap)parser.getJsonConfigs(); + lastJSONConfigTS = timestamp; + update = true; + LOGGER.debug("update is in order - updating timestamp: " + lastJSONConfigTS); + } + } catch (JsonProcessingException e) { + LOGGER.error("error processing JSON response: " + e); + } catch (IOException e) { + LOGGER.error("unable to collect remote JMX configs: " + e); + } + + return update; + } + private void reportStatus(AppConfig appConfig, Reporter reporter, Instance instance, int metricCount, String message, String status) { String checkName = instance.getCheckName(); @@ -521,17 +584,50 @@ private void sendServiceCheck(Reporter reporter, Instance instance, String messa reporter.resetServiceCheckCount(checkName); } + private void instantiate(LinkedHashMap instanceMap, LinkedHashMap initConfig, + String checkName, AppConfig appConfig, boolean forceNewConnection) { + + Instance instance; + Reporter reporter = appConfig.getReporter(); + + try { + instance = new Instance(instanceMap, initConfig, checkName, appConfig); + } catch (Exception e) { + String warning = "Unable to create instance. Please check your yaml file"; + appConfig.getStatus().addInitFailedCheck(checkName, warning, Status.STATUS_ERROR); + LOGGER.error(warning, e); + return; + } + + try { + // initiate the JMX Connection + instance.init(forceNewConnection); + instances.add(instance); + } catch (IOException e) { + instance.cleanUp(); + brokenInstances.add(instance); + String warning = CANNOT_CONNECT_TO_INSTANCE + instance + ". " + e.getMessage(); + this.reportStatus(appConfig, reporter, instance, 0, warning, Status.STATUS_ERROR); + this.sendServiceCheck(reporter, instance, warning, Status.STATUS_ERROR); + LOGGER.error(warning, e); + } catch (Exception e) { + instance.cleanUp(); + brokenInstances.add(instance); + String warning = "Unexpected exception while initiating instance " + instance + " : " + e.getMessage(); + this.reportStatus(appConfig, reporter, instance, 0, warning, Status.STATUS_ERROR); + this.sendServiceCheck(reporter, instance, warning, Status.STATUS_ERROR); + LOGGER.error(warning, e); + } + } + public void init(boolean forceNewConnection) { clearInstances(instances); clearInstances(brokenInstances); - Reporter reporter = appConfig.getReporter(); - Iterator> it = configs.entrySet().iterator(); - // SD config cache doesn't remove configs - it just overwrites. - Iterator> itSD = adConfigs.entrySet().iterator(); + Iterator> itSD = adPipeConfigs.entrySet().iterator(); while (it.hasNext() || itSD.hasNext()) { Map.Entry entry; boolean sdIterator = false; @@ -544,6 +640,7 @@ public void init(boolean forceNewConnection) { String name = entry.getKey(); YamlParser yamlConfig = entry.getValue(); + // AD config cache doesn't remove configs - it just overwrites. if(!sdIterator) { it.remove(); } @@ -557,35 +654,20 @@ public void init(boolean forceNewConnection) { } for (LinkedHashMap configInstance : configInstances) { - Instance instance; //Create a new Instance object - try { - instance = new Instance(configInstance, (LinkedHashMap) yamlConfig.getInitConfig(), - name, appConfig); - } catch (Exception e) { - String warning = "Unable to create instance. Please check your yaml file"; - appConfig.getStatus().addInitFailedCheck(name, warning, Status.STATUS_ERROR); - LOGGER.error(warning, e); - continue; - } - try { - // initiate the JMX Connection - instance.init(forceNewConnection); - instances.add(instance); - } catch (IOException e) { - instance.cleanUp(); - brokenInstances.add(instance); - String warning = CANNOT_CONNECT_TO_INSTANCE + instance + ". " + e.getMessage(); - this.reportStatus(appConfig, reporter, instance, 0, warning, Status.STATUS_ERROR); - this.sendServiceCheck(reporter, instance, warning, Status.STATUS_ERROR); - LOGGER.error(warning, e); - } catch (Exception e) { - instance.cleanUp(); - brokenInstances.add(instance); - String warning = "Unexpected exception while initiating instance " + instance + " : " + e.getMessage(); - this.reportStatus(appConfig, reporter, instance, 0, warning, Status.STATUS_ERROR); - this.sendServiceCheck(reporter, instance, warning, Status.STATUS_ERROR); - LOGGER.error(warning, e); + instantiate(configInstance, (LinkedHashMap) yamlConfig.getInitConfig(), + name, appConfig, forceNewConnection); + } + } + + //Process JSON configurations + if (adJSONConfigs != null) { + for (String check : adJSONConfigs.keySet()) { + HashMap checkConfig = (HashMap) adJSONConfigs.get(check); + LinkedHashMap initConfig = (LinkedHashMap) checkConfig.get("init_config"); + ArrayList> configInstances = (ArrayList>) checkConfig.get("instances"); + for (LinkedHashMap configInstance : configInstances) { + instantiate(configInstance, initConfig, check, appConfig, forceNewConnection); } } } diff --git a/src/main/java/org/datadog/jmxfetch/AppConfig.java b/src/main/java/org/datadog/jmxfetch/AppConfig.java index 72c4ebefc..92501a65d 100644 --- a/src/main/java/org/datadog/jmxfetch/AppConfig.java +++ b/src/main/java/org/datadog/jmxfetch/AppConfig.java @@ -5,7 +5,6 @@ import org.datadog.jmxfetch.converter.ExitWatcherConverter; import org.datadog.jmxfetch.converter.ReporterConverter; -import org.datadog.jmxfetch.converter.StatusConverter; import org.datadog.jmxfetch.reporter.ConsoleReporter; import org.datadog.jmxfetch.reporter.Reporter; import org.datadog.jmxfetch.validator.Log4JLevelValidator; @@ -90,9 +89,8 @@ class AppConfig { @Parameter(names = {"--status_location", "-s"}, description = "Absolute path of the status file. (default to null = no status file written)", - converter = StatusConverter.class, required = false) - private Status status = new Status(); + private String statusLocation; @Parameter(names = {"--exit_file_location", "-e"}, description = "Absolute path of the trigger file to watch to exit. (default to null = no exit on file)", @@ -106,12 +104,39 @@ class AppConfig { required = true) private List action = null; + @Parameter(names = {"--ipc_host", "-H"}, + description = "IPC host", + required = false) + private String ipcHost; + @Parameter(names = {"--ipc_port", "-I"}, description = "IPC port", validateWith = PositiveIntegerValidator.class, required = false) private int ipcPort = 0; + private Status status = new Status(); + + public boolean updateStatus() { + if (statusLocation != null) { + status = new Status(statusLocation); + return true; + } else if (ipcHost != null && ipcPort > 0) { + status = new Status(ipcHost, ipcPort); + return true; + } + + return false; + } + + public boolean remoteEnabled() { + return (ipcHost != null && ipcPort > 0); + } + + public String getStatusLocation() { + return this.statusLocation; + } + public String getAction() { return this.action.get(0); } @@ -136,11 +161,15 @@ public int getCheckPeriod() { return checkPeriod; } + public String getIPCHost() { + return ipcHost; + } + public int getIPCPort() { return ipcPort; } - public boolean getAutoDiscoveryEnabled() { + public boolean getAutoDiscoveryPipeEnabled() { return adEnabled; } diff --git a/src/main/java/org/datadog/jmxfetch/HttpClient.java b/src/main/java/org/datadog/jmxfetch/HttpClient.java new file mode 100644 index 000000000..9cc99d778 --- /dev/null +++ b/src/main/java/org/datadog/jmxfetch/HttpClient.java @@ -0,0 +1,135 @@ +package org.datadog.jmxfetch; + +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.URL; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import javax.net.ssl.HttpsURLConnection; + + +public class HttpClient { + private String token; + private TrustManager[] dummyTrustManager; + private SSLContext sc; + private String host; + private int port; + + private final static String USER_AGENT = "Datadog/JMXFetch"; + private final static Logger LOGGER = Logger.getLogger(Status.class.getName()); + + public static class HttpResponse { + private int responseCode; + private String responseBody; + + public HttpResponse(int responseCode, String responseBody) { + this.responseCode = responseCode; + this.responseBody = responseBody; + } + + public HttpResponse(int responseCode, InputStreamReader responseStream) throws IOException { + String inputLine; + BufferedReader in = new BufferedReader(responseStream); + StringBuffer responseBuilder = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) { + responseBuilder.append(inputLine); + } + in.close(); + + this.responseCode = responseCode; + this.responseBody = responseBuilder.toString(); + } + + public void setResponseCode(int responseCode) { + this.responseCode = responseCode; + } + public int getResponseCode() { + return this.responseCode; + } + public String getResponseBody() { + return this.responseBody; + } + public boolean isResponse2xx() { + return (responseCode >= 200 && responseCode <300); + } + } + + public HttpClient(String host, int port, boolean verify) { + this.host = host; + this.port = port; + this.token = System.getenv("SESSION_TOKEN"); + + if (!verify) { + try { + dummyTrustManager = new TrustManager[] { + new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + } + }; + sc = SSLContext.getInstance("SSL"); + sc.init(null, this.dummyTrustManager, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (Exception e) { + LOGGER.debug("session token unavailable - not setting"); + this.token = ""; + } + } + } + + /** + * only supports json bodies for now. + * */ + public HttpResponse request(String method, String body, String path) { + HttpClient.HttpResponse response = new HttpClient.HttpResponse(0, ""); + try { + String url = "https://"+ host + ":" + port + "/" + path; + LOGGER.debug("attempting to connect to: " + url); + LOGGER.debug("with body: " + body); + + URL uri = new URL(url); + HttpsURLConnection con = (HttpsURLConnection) uri.openConnection(); + + //add request header + con.setRequestMethod(method.toUpperCase()); + con.setRequestProperty("Authorization", "Bearer "+ this.token); + con.setRequestProperty("User-Agent", USER_AGENT); + if (method.toUpperCase() == "GET") { + con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + } else { + con.setRequestProperty("Content-Type", "application/json"); + con.setDoOutput(true); + DataOutputStream wr = new DataOutputStream(con.getOutputStream()); + wr.writeBytes(body); + wr.flush(); + wr.close(); + } + + int responseCode = con.getResponseCode(); + if(responseCode<200 || responseCode>=300) { + LOGGER.debug("HTTP error stream: " + con.getErrorStream()); + response.setResponseCode(responseCode); + } else { + response = new HttpClient.HttpResponse(responseCode, new InputStreamReader(con.getInputStream())); + } + + } catch (Exception e) { + LOGGER.info("problem creating http request: " + e.toString()); + } + + return response; + } +} diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index fc807d33e..85b45609b 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -42,7 +42,7 @@ public class Instance { private long lastCollectionTime; private Integer minCollectionPeriod; private long lastRefreshTime; - private LinkedHashMap yaml; + private LinkedHashMap instanceMap; private LinkedHashMap initConfig; private String instanceName; private LinkedHashMap tags; @@ -55,8 +55,8 @@ public class Instance { public Instance(Instance instance, AppConfig appConfig) { - this(instance.getYaml() != null - ? new LinkedHashMap(instance.getYaml()) + this(instance.getInstanceMap() != null + ? new LinkedHashMap(instance.getInstanceMap()) : null, instance.getInitConfig() != null ? new LinkedHashMap(instance.getInitConfig()) @@ -66,31 +66,31 @@ public Instance(Instance instance, AppConfig appConfig) { } @SuppressWarnings("unchecked") - public Instance(LinkedHashMap yamlInstance, LinkedHashMap initConfig, + public Instance(LinkedHashMap instanceMap, LinkedHashMap initConfig, String checkName, AppConfig appConfig) { this.appConfig = appConfig; - this.yaml = yamlInstance != null ? new LinkedHashMap(yamlInstance) : null; + this.instanceMap = instanceMap != null ? new LinkedHashMap(instanceMap) : null; this.initConfig = initConfig != null ? new LinkedHashMap(initConfig) : null; - this.instanceName = (String) yaml.get("name"); - this.tags = getTagsMap(yaml.get("tags")); + this.instanceName = (String) instanceMap.get("name"); + this.tags = getTagsMap(instanceMap.get("tags")); this.checkName = checkName; this.matchingAttributes = new LinkedList(); this.failingAttributes = new HashSet(); - this.refreshBeansPeriod = (Integer) yaml.get("refresh_beans"); + this.refreshBeansPeriod = (Integer) instanceMap.get("refresh_beans"); if (this.refreshBeansPeriod == null) { this.refreshBeansPeriod = DEFAULT_REFRESH_BEANS_PERIOD; // Make sure to refresh the beans list every 10 minutes // Useful because sometimes if the application restarts, jmxfetch might read // a jmxtree that is not completely initialized and would be missing some attributes } - this.minCollectionPeriod = (Integer) yaml.get("min_collection_interval"); + this.minCollectionPeriod = (Integer) instanceMap.get("min_collection_interval"); if (this.minCollectionPeriod == null && initConfig != null) { this.minCollectionPeriod = (Integer) initConfig.get("min_collection_interval"); } this.lastCollectionTime = 0; this.lastRefreshTime = 0; this.limitReached = false; - Object maxReturnedMetrics = this.yaml.get("max_returned_metrics"); + Object maxReturnedMetrics = this.instanceMap.get("max_returned_metrics"); if (maxReturnedMetrics == null) { this.maxReturnedMetrics = MAX_RETURNED_METRICS; } else { @@ -99,10 +99,10 @@ public Instance(LinkedHashMap yamlInstance, LinkedHashMap yamlInstance, LinkedHashMap conf : (ArrayList>) (yamlConf)) { + for (LinkedHashMap conf : (ArrayList>) (instanceConf)) { configurationList.add(new Configuration(conf)); } } @@ -139,16 +139,16 @@ public Instance(LinkedHashMap yamlInstance, LinkedHashMap getTagsMap(Object yamlTags){ + private static LinkedHashMap getTagsMap(Object tagsMap){ try { // Input has `Map` format - return (LinkedHashMap) yamlTags; + return (LinkedHashMap) tagsMap; } catch (ClassCastException e){ // Input has `List` format LinkedHashMap tags = new LinkedHashMap(); - for (String tag: (List)yamlTags) { + for (String tag: (List)tagsMap) { tags.put(tag, null); } @@ -187,7 +187,7 @@ public Connection getConnection(LinkedHashMap connectionParams, public void init(boolean forceNewConnection) throws IOException, FailedLoginException, SecurityException { LOGGER.info("Trying to connect to JMX Server at " + this.toString()); - connection = getConnection(yaml, forceNewConnection); + connection = getConnection(instanceMap, forceNewConnection); LOGGER.info("Connected to JMX Server at " + this.toString()); this.refreshBeansList(); this.getMatchingAttributes(); @@ -195,19 +195,19 @@ public void init(boolean forceNewConnection) throws IOException, FailedLoginExce @Override public String toString() { - if (this.yaml.get(PROCESS_NAME_REGEX) != null) { - return "process_regex: `" + this.yaml.get(PROCESS_NAME_REGEX) + "`"; - } else if (this.yaml.get("jmx_url") != null) { - return (String) this.yaml.get("jmx_url"); + if (this.instanceMap.get(PROCESS_NAME_REGEX) != null) { + return "process_regex: `" + this.instanceMap.get(PROCESS_NAME_REGEX) + "`"; + } else if (this.instanceMap.get("jmx_url") != null) { + return (String) this.instanceMap.get("jmx_url"); } else { - return this.yaml.get("host") + ":" + this.yaml.get("port"); + return this.instanceMap.get("host") + ":" + this.instanceMap.get("port"); } } public LinkedList> getMetrics() throws IOException { // We can force to refresh the bean list every x seconds in case of ephemeral beans - // To enable this, a "refresh_beans" parameter must be specified in the yaml config file + // To enable this, a "refresh_beans" parameter must be specified in the yaml/json config if (this.refreshBeansPeriod != null && (System.currentTimeMillis() - this.lastRefreshTime) / 1000 > this.refreshBeansPeriod) { LOGGER.info("Refreshing bean list"); this.refreshBeansList(); @@ -216,8 +216,7 @@ public LinkedList> getMetrics() throws IOException { LinkedList> metrics = new LinkedList>(); Iterator it = matchingAttributes.iterator(); - - + // increment the lastCollectionTime this.lastCollectionTime = System.currentTimeMillis(); @@ -392,8 +391,8 @@ private void refreshBeansList() throws IOException { public String[] getServiceCheckTags() { List tags = new ArrayList(); - if (this.yaml.get("host") != null) { - tags.add("jmx_server:" + this.yaml.get("host")); + if (this.instanceMap.get("host") != null) { + tags.add("jmx_server:" + this.instanceMap.get("host")); } if (this.tags != null) { for (Entry e : this.tags.entrySet()) { @@ -412,8 +411,8 @@ public String getName() { return this.instanceName; } - LinkedHashMap getYaml() { - return this.yaml; + LinkedHashMap getInstanceMap() { + return this.instanceMap; } LinkedHashMap getInitConfig() { diff --git a/src/main/java/org/datadog/jmxfetch/JsonParser.java b/src/main/java/org/datadog/jmxfetch/JsonParser.java new file mode 100644 index 000000000..b54fa6f1e --- /dev/null +++ b/src/main/java/org/datadog/jmxfetch/JsonParser.java @@ -0,0 +1,50 @@ +package org.datadog.jmxfetch; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.IOException; +import java.util.HashMap; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; + +@SuppressWarnings("unchecked") +class JsonParser { + + private HashMap parsedJson; + + public JsonParser(InputStream jsonInputStream) throws IOException{ + ObjectMapper mapper = new ObjectMapper(); + InputStreamReader jsonInputStreamReader = new InputStreamReader(jsonInputStream); + parsedJson = mapper.readValue(jsonInputStreamReader, new TypeReference(){}); + } + + public JsonParser(JsonParser other) { + parsedJson = new HashMap((HashMap) other.getParsedJson()); + } + + public Object getJsonConfigs() { + return parsedJson.get("configs"); + } + + public Object getJsonTimestamp() { + return parsedJson.get("timestamp"); + } + + public Object getJsonInstances(String key) { + HashMap config = (HashMap) ((HashMap) parsedJson + .get("configs")).get(key); + return config.get("instances"); + } + + public Object getInitConfig(String key) { + HashMap config = (HashMap) ((HashMap) parsedJson + .get("configs")).get(key); + return config.get("init_config"); + } + + public Object getParsedJson() { + return parsedJson; + } + +} diff --git a/src/main/java/org/datadog/jmxfetch/Status.java b/src/main/java/org/datadog/jmxfetch/Status.java index 41612aa89..53a421158 100644 --- a/src/main/java/org/datadog/jmxfetch/Status.java +++ b/src/main/java/org/datadog/jmxfetch/Status.java @@ -4,20 +4,14 @@ import java.util.HashMap; import java.util.LinkedList; import java.io.BufferedReader; -import java.io.DataOutputStream; import java.io.InputStreamReader; -import java.net.URL; import java.lang.System; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import javax.net.ssl.HttpsURLConnection; - import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.yaml.snakeyaml.Yaml; -import com.google.gson.Gson; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonProcessingException; public class Status { @@ -27,46 +21,31 @@ public class Status { private final static Logger LOGGER = Logger.getLogger(Status.class.getName()); private final static String INITIALIZED_CHECKS = "initialized_checks"; private final static String FAILED_CHECKS = "failed_checks"; + private final static String API_STATUS_PATH = "agent/jmx/status"; private HashMap instanceStats; - private TrustManager[] dummyTrustManager; - private SSLContext sc; + private ObjectMapper mapper; private String statusFileLocation; + private HttpClient client; private boolean isEnabled; - private String token; public Status() { this(null); } + public Status(String host, int port) { + mapper = new ObjectMapper(); + client = new HttpClient(host, port, false); + configure(null, host, port); + } + public Status(String statusFileLocation) { - configure(statusFileLocation); + configure(statusFileLocation, null, 0); } - void configure(String statusFileLocation) { + void configure(String statusFileLocation, String host, int port) { this.statusFileLocation = statusFileLocation; this.instanceStats = new HashMap(); - try { - this.token = System.getenv("SESSION_TOKEN"); - dummyTrustManager = new TrustManager[] { - new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { - } - - public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { - } - } - }; - sc = SSLContext.getInstance("SSL"); - sc.init(null, this.dummyTrustManager, new java.security.SecureRandom()); - HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); - } catch (Exception e) { - LOGGER.debug("session token unavailable - not setting"); - this.token = ""; - } - this.isEnabled = (this.statusFileLocation != null || this.token != ""); + this.isEnabled = (this.statusFileLocation != null || this.client != null); this.clearStats(); } @@ -122,47 +101,20 @@ private String generateYaml() { return yaml.dump(status); } - private String generateJson() { - Gson gson = new Gson(); + private String generateJson() throws JsonProcessingException { HashMap status = new HashMap(); status.put("timestamp", System.currentTimeMillis()); status.put("checks", this.instanceStats); - return gson.toJson(status); - } - - private boolean postRequest(String body, int port) { - int responseCode = 0; - try { - String url = "https://localhost:" + port + "/agent/jmxstatus"; - - URL uri = new URL(url); - HttpsURLConnection con = (HttpsURLConnection) uri.openConnection(); - - //add reuqest header - con.setRequestMethod("POST"); - con.setRequestProperty("Content-Type", "application/json"); - con.setRequestProperty("Authorization", "Bearer "+ this.token); - - con.setDoOutput(true); - DataOutputStream wr = new DataOutputStream(con.getOutputStream()); - wr.writeBytes(body); - wr.flush(); - wr.close(); - - responseCode = con.getResponseCode(); - - } catch (Exception e) { - LOGGER.info("problem creating http request: " + e.toString()); - } - return (responseCode >= 200 && responseCode < 300); + return mapper.writeValueAsString(status); } - public void flush(int port) { + public void flush() { if (isEnabled()) { - if (port > 0) { - String json = generateJson(); + if (this.client != null) { try { - if (!this.postRequest(json, port)) { + String json = generateJson(); + HttpClient.HttpResponse response = this.client.request("POST", json, API_STATUS_PATH); + if (!response.isResponse2xx()) { LOGGER.debug("Problem submitting JSON status: " + json); } } catch (Exception e) { diff --git a/src/main/java/org/datadog/jmxfetch/converter/StatusConverter.java b/src/main/java/org/datadog/jmxfetch/converter/StatusConverter.java deleted file mode 100644 index 16a144595..000000000 --- a/src/main/java/org/datadog/jmxfetch/converter/StatusConverter.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.datadog.jmxfetch.converter; - -import org.datadog.jmxfetch.Status; - -import com.beust.jcommander.IStringConverter; - -public class StatusConverter implements IStringConverter { - - public Status convert(String value) { - return new Status(value); - } -} \ No newline at end of file diff --git a/src/test/java/org/datadog/jmxfetch/TestConfiguration.java b/src/test/java/org/datadog/jmxfetch/TestConfiguration.java index a8118e0a4..5935624f0 100644 --- a/src/test/java/org/datadog/jmxfetch/TestConfiguration.java +++ b/src/test/java/org/datadog/jmxfetch/TestConfiguration.java @@ -6,6 +6,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -19,6 +20,7 @@ public class TestConfiguration { static LinkedList configurations = new LinkedList(); + static JsonParser adConfigs; /** * Setup Configuration tests @@ -26,7 +28,7 @@ public class TestConfiguration { */ @SuppressWarnings("unchecked") @BeforeClass - public static void init() throws FileNotFoundException { + public static void init() throws FileNotFoundException, IOException { File f = new File("src/test/resources/", "jmx_bean_scope.yaml"); String yamlPath = f.getAbsolutePath(); FileInputStream yamlInputStream = new FileInputStream(yamlPath); @@ -39,6 +41,39 @@ public static void init() throws FileNotFoundException { configurations.add(new Configuration(conf)); } } + + // lets also collect auto-discovery configs + f = new File("src/test/resources/", "auto_discovery_configs.json"); + String jsonPath = f.getAbsolutePath(); + FileInputStream jsonInputStream = new FileInputStream(jsonPath); + adConfigs = new JsonParser(jsonInputStream); + } + + /** + * Stringify a bean pattern to comply with the representation of a MBean + * @throws SecurityException + * @throws NoSuchMethodException + * @throws InvocationTargetException + * @throws IllegalArgumentException + * @throws IllegalAccessException + */ + @Test + public void testAutoDiscoveryConfigs() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException{ + HashMap configs = (HashMap) adConfigs.getJsonConfigs(); + + assertEquals(configurations.size(), 4); + int nconfigs = 0; + for (String check : configs.keySet()) { + ArrayList> configInstances = ((ArrayList>) adConfigs.getJsonInstances(check)); + for (LinkedHashMap config : configInstances) { + Object jsonConf = config.get("conf"); + for (LinkedHashMap conf : (ArrayList>) (jsonConf)) { + configurations.add(new Configuration(conf)); + nconfigs++; + } + } + } + assertEquals(configurations.size(), 4 + nconfigs); } /** @@ -91,7 +126,7 @@ public void testCommonBeanKeys() throws FileNotFoundException, NoSuchMethodExcep HashMap> parametersIntersectionByDomain = (HashMap>) getCommonBeanKeysByDomain.invoke(null, filtersByDomain); // Only contains 'org.datadog.jmxfetch.test' domain - assertEquals(parametersIntersectionByDomain.size(), 1); + assertEquals(parametersIntersectionByDomain.size(), 2); assertTrue(parametersIntersectionByDomain.containsKey("org.datadog.jmxfetch.test")); // Parameters intersection should match: 'param', 'scope' and 'type' @@ -129,7 +164,7 @@ public void testCommonScope() throws NoSuchMethodException, SecurityException, I HashMap> commonBeanScopeByDomain = (HashMap>) getCommonScopeByDomain.invoke(null, parametersIntersectionByDomain, filtersByDomain); // Only contains 'org.datadog.jmxfetch.test' domain - assertEquals(commonBeanScopeByDomain.size(), 1); + assertEquals(commonBeanScopeByDomain.size(), 2); assertTrue(commonBeanScopeByDomain.containsKey("org.datadog.jmxfetch.test")); LinkedHashMap beanScope = commonBeanScopeByDomain.get("org.datadog.jmxfetch.test"); diff --git a/src/test/java/org/datadog/jmxfetch/TestParsingJCommander.java b/src/test/java/org/datadog/jmxfetch/TestParsingJCommander.java index 9760f1018..cd10efbc2 100644 --- a/src/test/java/org/datadog/jmxfetch/TestParsingJCommander.java +++ b/src/test/java/org/datadog/jmxfetch/TestParsingJCommander.java @@ -26,6 +26,8 @@ public class TestParsingJCommander { private static final List MULTI_CHECK = Arrays.asList("jmx.yaml", "jmx-2.yaml"); private static final String STATUS_LOCATION = "/status/status_location"; private static final String EXIT_FILE_LOCATION = "/status/exit_locationt"; + private static final String IPC_HOSTNAME = "localhost"; + private static final String IPC_PORT = "5001"; private static AppConfig testCommand(String[] params) throws ParameterException { AppConfig appConfig = new AppConfig(); @@ -261,11 +263,29 @@ public void testParsingStatus() { AppConfig.ACTION_COLLECT }; AppConfig appConfig = testCommand(params); + assertTrue(appConfig.updateStatus()); assertNotNull(appConfig.getStatus()); assertEquals(STATUS_LOCATION, appConfig.getStatus().getStatusFileLocation()); assertTrue(appConfig.getStatus().isEnabled()); } + @Test + public void testParsingStatusIPC() { + String[] params = new String[]{ + "--reporter", REPORTER_CONSOLE, + "--check", SINGLE_CHECK, + "--conf_directory", CONF_DIR, + "--ipc_host", IPC_HOSTNAME, + "--ipc_port", IPC_PORT, + AppConfig.ACTION_COLLECT + }; + AppConfig appConfig = testCommand(params); + assertTrue(appConfig.updateStatus()); + + assertNotNull(appConfig.getStatus()); + assertTrue(appConfig.getStatus().isEnabled()); + } + @Test public void testParsingExitWatcher() { String[] params = new String[]{ diff --git a/src/test/resources/auto_discovery_configs.json b/src/test/resources/auto_discovery_configs.json new file mode 100644 index 000000000..197f8f085 --- /dev/null +++ b/src/test/resources/auto_discovery_configs.json @@ -0,0 +1,85 @@ +{ + "configs": { + "jmx": { + "instances": [ + { + "process_name_regex": ".*surefire.*", + "name": "jmx_test_instance", + "conf": [ + { + "include": { + "scope": "sameScope", + "domain": "org.datadog.jmxfetch.test", + "type": "sameType", + "additional": "additionalParam", + "param": "sameParam" + } + }, + { + "include": { + "scope": "sameScope", + "domain": "org.datadog.jmxfetch.test", + "type": "sameType", + "param": "sameParam" + } + }, + { + "include": { + "scope": "sameScope", + "domain": "org.datadog.jmxfetch.test", + "type": [ + "sameType", + "notTheSameType" + ], + "param": "sameParam" + } + }, + { + "include": { + "bean": [ + "org.datadog.jmxfetch.test:scope=sameScope,param=sameParam,type=sameType", + "org.datadog.jmxfetch.test:scope=sameScope,param=notTheSameParam,type=sameType" + ] + } + } + ] + } + ], + "init_config": null + }, + "cassandra": { + "instances": [ + { + "process_name_regex": ".*surefire.*", + "name": "jmx_first_instance", + "conf": [ + { + "include": { + "attribute": [ + "ShouldBe100" + ], + "bean": "org.apache.cassandra.metrics:keyspace=MyKeySpace,type=ColumnFamily,scope=MyColumnFamily,name=PendingTasks" + } + } + ], + "cassandra_aliasing": true + }, + { + "process_name_regex": ".*surefire.*", + "name": "jmx_second_instance", + "conf": [ + { + "include": { + "attribute": [ + "ShouldBe1000" + ], + "bean": "org.apache.cassandra.metrics:keyspace=MyKeySpace,type=ColumnFamily,scope=MyColumnFamily,name=PendingTasks" + } + } + ] + } + ], + "init_config": null + } + } +} \ No newline at end of file