From 0817de2aa73939abc60427fe18ffa6cb71938eff Mon Sep 17 00:00:00 2001 From: "rohit.k" Date: Sun, 23 Apr 2017 19:37:07 +0530 Subject: [PATCH 01/16] Implementing dropwizard emitter for druid making metric manager and alert emitters as optional --- docs/design/extensions-contrib/dropwizard.md | 100 +++++++ extensions-contrib/dropwizard-emitter/pom.xml | 91 +++++++ .../emitter/dropwizard/DropwizardEmitter.java | 108 ++++++++ .../dropwizard/DropwizardEmitterConfig.java | 91 +++++++ .../dropwizard/DropwizardEmitterModule.java | 70 +++++ .../emitter/dropwizard/DropwizardEvent.java | 38 +++ .../dropwizard/DropwizardMetricManager.java | 32 +++ .../dropwizard/DropwizardReporter.java | 39 +++ .../DruidToDropwizardEventConverter.java | 43 +++ .../dropwizard/HistogramMetricManager.java | 46 ++++ .../SendAllDropwizardEventConverter.java | 132 +++++++++ .../WhiteListBasedConverter.java | 253 ++++++++++++++++++ .../reporters/DropwizardConsoleReporter.java | 120 +++++++++ .../reporters/DropwizardJMXReporter.java | 46 ++++ .../io.druid.initialization.DruidModule | 1 + .../main/resources/defaultWhiteListMap.json | 48 ++++ .../DropwizardEmitterConfigTest.java | 63 +++++ 17 files changed, 1321 insertions(+) create mode 100644 docs/design/extensions-contrib/dropwizard.md create mode 100644 extensions-contrib/dropwizard-emitter/pom.xml create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule create mode 100644 extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json create mode 100644 extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md new file mode 100644 index 000000000000..17f248239c47 --- /dev/null +++ b/docs/design/extensions-contrib/dropwizard.md @@ -0,0 +1,100 @@ +--- +layout: doc_page +--- + +# Dropwizard Emitter + +To use this extension, make sure to [include](../../operations/including-extensions.html) `dropwizard-emitter` extension. + +## Introduction + +The intent of this extension is to integrate [Dropwizard](http://metrics.dropwizard.io/3.1.0/getting-started/#) metrics library with druid so that dropwizard users can easily absorb druid into their monitoring ecosystem. +It accumulates druid metrics in a dropwizard histogram and emits them to various sinks via dropwizard supported reporters. +A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, etc., it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles. +Currently dropwizard metrics can be emitted to these sinks: +Console,HTTP,JMX,Graphite,CSV,Slf4jLogger,Ganglia and various other community supported [sinks](http://metrics.dropwizard.io/3.1.0/manual/third-party/). + +## Configuration + +All the configuration parameters for Dropwizard emitter are under `druid.emitter.dropwizard`. + +|property|description|required?|default| +|--------|-----------|---------|-------| +|`druid.emitter.dropwizard.metric`|The metric manager to be used.Currently supported metric manager is histogram|no|histogram| +|`druid.emitter.dropwizard.reporter`|The dropwizard reporter to be used.|yes|none| +|`druid.emitter.dropwizard.eventConverter`| Filter and converter of druid events to dropwizard event(please see next section). |yes|none| +|`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| + + +### Druid to Dropwizard Event Converter + +Dropwizard Event Converter defines a mapping between druid metrics name plus dimensions to a Dropwizard metric name. +Dropwizard metric name is organized using the following schema: +`.[].[]..` +Properly naming the metrics is critical to avoid conflicts, confusing data and potentially wrong interpretation later on. + +Example `druid.historical.abc_com:8080.MyDataSourceName.GroupBy.query/time`: + + * `druid` -> namespace prefix + * `historical` -> service name + * `abc.com:8080` -> druid hostname + * `MyDataSourceName` -> dimension value + * `GroupBy` -> dimension value + * `query/time` -> metric name + +We have two different implementation of event converter: + +#### Send-All converter + +The first implementation called `all`, will send all the druid service metrics events. +The metric name will be in the form `.[].[]..` +User has control of `.[].[].` + +You can omit the hostname by setting `ignoreHostname=true` +`druid.SERVICE_NAME.dataSourceName.queryType.query.time` + +You can omit the service name by setting `ignoreServiceName=true` +`druid.HOSTNAME.dataSourceName.queryType.query.time` + +```json + +druid.emitter.dropwizard.eventConverter={"type":"all", "namespacePrefix": "druid.test", "ignoreHostname":true, "ignoreServiceName":true} + +``` + +#### White-list based converter + +The second implementation called `whiteList`, will send only the white listed metrics and dimensions. +Same as for the `all` converter user has control of `.[].[].` +White-list based converter comes with the following default white list map located under resources in `./src/main/resources/defaultWhiteListMap.json` + +Although user can override the default white list map by supplying a property called `mapPath`. +This property is a String containing the path for the file containing **white list map Json object**. +For example the following converter will read the map from the file `/pathPrefix/fileName.json`. + +```json + +druid.emitter.dropwizard.eventConverter={"type":"whiteList", "namespacePrefix": "druid.test", "ignoreHostname":true, "ignoreServiceName":true, "mapPath":"/pathPrefix/fileName.json"} + +``` + +**Druid emits a huge number of metrics we highly recommend to use the `whiteList` converter** + +### Metric Manager + +Metric manager defines the dropwizard accumulator that would be used to accumulate druid metric events. +For eg : Gauge,Counter,Histogram,Meter etc. + +### Dropwizard reporter + +```json + +druid.emitter.dropwizard.reporter={"type":"jmx"} + +``` + +```json + +druid.emitter.dropwizard.reporter={"type":"console","emitIntervalInSecs":30}"} + +``` \ No newline at end of file diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml new file mode 100644 index 000000000000..e0e1ebd3944a --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -0,0 +1,91 @@ + + + + + 4.0.0 + + + io.druid + druid + 0.10.0-SNAPSHOT + ../../pom.xml + + + io.druid.extensions.contrib + dropwizard-emitter + dropwizard-emitter + Druid emitter extension to convert druid metric to Dropwizard metrics + + + + io.druid + druid-common + ${project.parent.version} + provided + + + io.druid + druid-api + ${project.parent.version} + provided + + + com.metamx + emitter + provided + + + junit + junit + test + + + org.easymock + easymock + test + + + pl.pragmatists + JUnitParams + 1.0.4 + test + + + io.druid + druid-server + ${project.parent.version} + test-jar + test + + + io.druid + druid-processing + ${project.parent.version} + test-jar + test + + + io.dropwizard.metrics + metrics-core + 3.1.0 + + + diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java new file mode 100644 index 000000000000..153c03bbf071 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java @@ -0,0 +1,108 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.core.Event; +import com.metamx.emitter.service.AlertEvent; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.logger.Logger; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; + +public class DropwizardEmitter implements Emitter { + private static Logger log = new Logger(DropwizardEmitter.class); + private final MetricRegistry metricsRegistry = new MetricRegistry(); + private final DropwizardMetricManager dropwizardMetricManager; + private final AtomicBoolean started = new AtomicBoolean(false); + private final DruidToDropwizardEventConverter druidToDropwizardEventConverter; + private final List emitterList; + //TODO make this a list of reporters + private final DropwizardReporter dropwizardReporter; + + public DropwizardEmitter(DropwizardEmitterConfig dropwizardEmitterConfig,List emitterList) { + this.dropwizardMetricManager = dropwizardEmitterConfig.getDropwizardMetricManager(); + this.druidToDropwizardEventConverter = dropwizardEmitterConfig.getDruidToDropwizardEventConverter(); + this.emitterList = emitterList; + this.dropwizardReporter = dropwizardEmitterConfig.getDropwizardReporter(); + } + + @Override + public void start() { + try { + startReporters(); + } catch (IOException e) { + log.error(e,"Error while starting Dropwizard reporters"); + } + started.set(true); + } + + @Override + public void emit(Event event) { + if (!started.get()) { + throw new ISE("Emit was called while emitter is yet not initialized"); + } + if (event instanceof ServiceMetricEvent) { + DropwizardEvent dropwizardEvent = druidToDropwizardEventConverter.druidEventToDropwizard((ServiceMetricEvent) event); + if(dropwizardEvent!=null) { + dropwizardMetricManager.updateMetric(metricsRegistry,dropwizardEvent); + }else{ + log.debug("Dropping the service event "+event); + return; + } + } else if (!emitterList.isEmpty() && event instanceof AlertEvent) { + for (Emitter emitter : emitterList) { + emitter.emit(event); + } + } else if (event instanceof AlertEvent) { + AlertEvent alertEvent = (AlertEvent) event; + log.error( + "The following alert is dropped, description is [%s], severity is [%s]", + alertEvent.getDescription(), alertEvent.getSeverity() + ); + } else { + log.error("unknown event type [%s]", event.getClass()); + } + } + + @Override + public void flush() throws IOException { + dropwizardReporter.flush(); + } + + @Override + public void close() throws IOException { + dropwizardReporter.close(); + } + + private void startReporters() throws IOException { + dropwizardReporter.start(metricsRegistry); + } + + public static String sanitize(String namespace) + { + Pattern DOT_OR_WHITESPACE = Pattern.compile("[\\s]+|[.]+"); + return DOT_OR_WHITESPACE.matcher(namespace).replaceAll("_"); + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java new file mode 100644 index 000000000000..5a642cc18689 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java @@ -0,0 +1,91 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collections; +import java.util.List; + + +public class DropwizardEmitterConfig { + @JsonProperty("eventConverter") + final private DruidToDropwizardEventConverter druidToDropwizardEventConverter; + @JsonProperty("reporter") + final private DropwizardReporter dropwizardReporter; + @JsonProperty("metric") + final private DropwizardMetricManager dropwizardMetricManager; + @JsonProperty + final private List alertEmitters; + + @JsonCreator + public DropwizardEmitterConfig(@JsonProperty("reporter") DropwizardReporter dropwizardReporter ,@JsonProperty("eventConverter") DruidToDropwizardEventConverter druidToDropwizardEventConverter,@JsonProperty("metric") DropwizardMetricManager dropwizardMetricManager,@JsonProperty("alertEmitters") List alertEmitters ) { + this.dropwizardReporter = dropwizardReporter; + this.druidToDropwizardEventConverter = druidToDropwizardEventConverter; + this.dropwizardMetricManager = dropwizardMetricManager==null? new HistogramMetricManager():dropwizardMetricManager; + this.alertEmitters = alertEmitters == null ? Collections.emptyList() : alertEmitters; + } + + + public DruidToDropwizardEventConverter getDruidToDropwizardEventConverter() { + return druidToDropwizardEventConverter; + } + + public DropwizardReporter getDropwizardReporter() { + return dropwizardReporter; + } + + public List getAlertEmitters() { + return alertEmitters; + } + + public DropwizardMetricManager getDropwizardMetricManager() { + return dropwizardMetricManager; + } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DropwizardEmitterConfig that = (DropwizardEmitterConfig) o; + + if (alertEmitters != null ? !alertEmitters.equals(that.alertEmitters) : that.alertEmitters != null) + return false; + if (dropwizardMetricManager != null ? !dropwizardMetricManager.equals(that.dropwizardMetricManager) : that.dropwizardMetricManager != null) + return false; + if (dropwizardReporter != null ? !dropwizardReporter.equals(that.dropwizardReporter) : that.dropwizardReporter != null) + return false; + if (druidToDropwizardEventConverter != null ? !druidToDropwizardEventConverter.equals(that.druidToDropwizardEventConverter) : that.druidToDropwizardEventConverter != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = druidToDropwizardEventConverter != null ? druidToDropwizardEventConverter.hashCode() : 0; + result = 31 * result + (dropwizardReporter != null ? dropwizardReporter.hashCode() : 0); + result = 31 * result + (dropwizardMetricManager != null ? dropwizardMetricManager.hashCode() : 0); + result = 31 * result + (alertEmitters != null ? alertEmitters.hashCode() : 0); + return result; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java new file mode 100644 index 000000000000..769acc3e0b9b --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java @@ -0,0 +1,70 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.metamx.emitter.core.Emitter; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.ManageLifecycle; +import io.druid.initialization.DruidModule; + +import java.util.Collections; +import java.util.List; + +public class DropwizardEmitterModule implements DruidModule +{ + private static final String EMITTER_TYPE = "dropwizard"; + @Override + public List getJacksonModules() + { + return Collections.EMPTY_LIST; + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter." + EMITTER_TYPE, DropwizardEmitterConfig.class); + + } + + @Provides + @ManageLifecycle + @Named(EMITTER_TYPE) + public Emitter getEmitter(DropwizardEmitterConfig dropwizardEmitterConfig, ObjectMapper mapper, final Injector injector){ + List emitters = Lists.transform( + dropwizardEmitterConfig.getAlertEmitters(), + new Function() { + @Override + public Emitter apply(String s) { + return injector.getInstance(Key.get(Emitter.class, Names.named(s))); + } + } + ); + return new DropwizardEmitter(dropwizardEmitterConfig, emitters); + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java new file mode 100644 index 000000000000..c32e9a8ab431 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java @@ -0,0 +1,38 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + + +public class DropwizardEvent { + private final String metricName; + private final Double value; + + public DropwizardEvent(String metricName, Double value) { + this.metricName = metricName; + this.value = value; + } + + public String getMetricName() { + return metricName; + } + + public Double getValue() { + return value; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java new file mode 100644 index 000000000000..d7a7fcf92210 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java @@ -0,0 +1,32 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = HistogramMetricManager.class) +@JsonSubTypes(value= { + @JsonSubTypes.Type(name="histogram", value = HistogramMetricManager.class) +}) + +public interface DropwizardMetricManager { + public void updateMetric(MetricRegistry metricsRegistry,DropwizardEvent dropwizardEvent); +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java new file mode 100644 index 000000000000..b2f39a65428f --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; +import io.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; + +import java.io.IOException; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value= { + @JsonSubTypes.Type(name="console", value = DropwizardConsoleReporter.class), + @JsonSubTypes.Type(name="jmx", value = DropwizardJMXReporter.class), +}) +public interface DropwizardReporter { + + public void start(MetricRegistry metricRegistry) throws IOException; + public void flush() throws IOException; + public void close() throws IOException; +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java new file mode 100644 index 000000000000..9042acb7bb9b --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java @@ -0,0 +1,43 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.emitter.dropwizard.eventconverters.SendAllDropwizardEventConverter; +import io.druid.emitter.dropwizard.eventconverters.WhiteListBasedConverter; + + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = SendAllDropwizardEventConverter.class) +@JsonSubTypes(value= { + @JsonSubTypes.Type(name="all", value = SendAllDropwizardEventConverter.class), + @JsonSubTypes.Type(name="whiteList", value = WhiteListBasedConverter.class) +}) +public interface DruidToDropwizardEventConverter { + /** + * This function acts as a filter. It returns null if the event is not suppose to be emitted to Dropwizard + * Also This function will define the conversion between the druid event dimension's values and Dropwizard's metric name + * + * @param serviceMetricEvent Druid event ot type {@link com.metamx.emitter.service.ServiceMetricEvent} + * + * @return {@link io.druid.emitter.dropwizard.DropwizardEvent} or null + */ + DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent); +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java new file mode 100644 index 000000000000..4ea72b5ce886 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java @@ -0,0 +1,46 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import io.druid.java.util.common.logger.Logger; +import io.druid.math.expr.Expr; + + +public class HistogramMetricManager implements DropwizardMetricManager { + private static Logger log = new Logger(HistogramMetricManager.class); + @Override + public void updateMetric(MetricRegistry metricsRegistry,DropwizardEvent dropwizardEvent) { + if(dropwizardEvent==null){ + log.error("Dropwizard event passed to " + getClass() + " is null"); + return; + } + metricsRegistry.histogram(dropwizardEvent.getMetricName()).update(dropwizardEvent.getValue().longValue()); + } + + @Override + public boolean equals(Object o){ + return true; + } + + @Override + public int hashCode() { + return 1; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java new file mode 100644 index 000000000000..dce204f15a7e --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java @@ -0,0 +1,132 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard.eventconverters; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedSet; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.emitter.dropwizard.DropwizardEmitter; +import io.druid.emitter.dropwizard.DropwizardEvent; +import io.druid.emitter.dropwizard.DruidToDropwizardEventConverter; + + +public class SendAllDropwizardEventConverter implements DruidToDropwizardEventConverter { + @JsonProperty + private boolean ignoreHostname; + @JsonProperty + private boolean ignoreServiceName; + @JsonProperty + private final String namespacePrefix; + + public String getNamespacePrefix() { + return namespacePrefix; + } + + @JsonCreator + public SendAllDropwizardEventConverter( + @JsonProperty("namespacePrefix") String namespacePrefix, + @JsonProperty("ignoreHostname") Boolean ignoreHostname, + @JsonProperty("ignoreServiceName") Boolean ignoreServiceName + ) + { + this.ignoreHostname = ignoreHostname == null ? false : ignoreHostname; + this.ignoreServiceName = ignoreServiceName == null ? false : ignoreServiceName; + this.namespacePrefix = namespacePrefix; + } + + @JsonProperty + public void setIgnoreHostname(boolean ignoreHostname) { + this.ignoreHostname = ignoreHostname; + } + + @JsonProperty + public void setIgnoreServiceName(boolean ignoreServiceName) { + this.ignoreServiceName = ignoreServiceName; + } + + @JsonProperty + public boolean isIgnoreServiceName() + { + return ignoreServiceName; + } + + @JsonProperty + public boolean isIgnoreHostname() + { + return ignoreHostname; + } + + @Override + public DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent) { + ImmutableList.Builder metricPathBuilder = new ImmutableList.Builder(); + + if (!Strings.isNullOrEmpty(this.getNamespacePrefix())) + metricPathBuilder.add(this.getNamespacePrefix()); + + if (!this.isIgnoreServiceName()) { + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getService())); + } + if (!this.isIgnoreHostname()) { + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getHost())); + } + + ImmutableSortedSet dimNames = ImmutableSortedSet.copyOf(serviceMetricEvent.getUserDims().keySet()); + for (String dimName : dimNames) { + metricPathBuilder.add(DropwizardEmitter.sanitize(String.valueOf(serviceMetricEvent.getUserDims() + .get(dimName)))); + } + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getMetric())); + + return new DropwizardEvent(Joiner.on(".").join(metricPathBuilder.build()), + serviceMetricEvent.getValue().doubleValue()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()){ + return false; + } + + SendAllDropwizardEventConverter that = (SendAllDropwizardEventConverter) o; + + if (ignoreHostname != that.ignoreHostname) { + return false; + } + if (ignoreServiceName != that.ignoreServiceName) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + int result = (ignoreHostname ? 1 : 0); + result = 31 * result + (ignoreServiceName ? 1 : 0); + return result; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java new file mode 100644 index 000000000000..59dfa6d16bb4 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java @@ -0,0 +1,253 @@ +/* +* Licensed to Metamarkets Group Inc. (Metamarkets) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. Metamarkets licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package io.druid.emitter.dropwizard.eventconverters; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.io.Files; +import com.google.common.io.Resources; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.emitter.dropwizard.DropwizardEmitter; +import io.druid.emitter.dropwizard.DropwizardEvent; +import io.druid.emitter.dropwizard.DruidToDropwizardEventConverter; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.logger.Logger; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.TimeUnit; + +@JsonTypeName("whiteList") +public class WhiteListBasedConverter implements DruidToDropwizardEventConverter { + private static final Logger LOGGER = new Logger(WhiteListBasedConverter.class); + /** + * @code whiteListDimsMapper is a white list of metric->dimensions mappings. + * Key is the metric name or the metric's prefix. + * Value is a list of metric's dimensions names. + * The order of the dimension name is important, it will be used to build the dropwizard metric nanme. + * For instance we have dimension type is nested under dimension dataSource -> prefix.dataSource.queryType.metricName + */ + private final ImmutableSortedMap> whiteListDimsMapper; + + @JsonProperty + private final boolean ignoreHostname; + + @JsonProperty + private final boolean ignoreServiceName; + + @JsonProperty + private final String namespacePrefix; + + @JsonProperty + private final String mapPath; + + private final ObjectMapper mapper; + + @JsonCreator + public WhiteListBasedConverter( + @JsonProperty("namespacePrefix") String namespacePrefix, + @JsonProperty("ignoreHostname") Boolean ignoreHostname, + @JsonProperty("ignoreServiceName") Boolean ignoreServiceName, + @JsonProperty("mapPath") String mapPath, + @JacksonInject ObjectMapper mapper + ) { + this.mapper = Preconditions.checkNotNull(mapper); + this.mapPath = mapPath; + this.whiteListDimsMapper = readMap(this.mapPath); + this.ignoreHostname = ignoreHostname == null ? false : ignoreHostname; + this.ignoreServiceName = ignoreServiceName == null ? false : ignoreServiceName; + this.namespacePrefix = namespacePrefix; + } + + @JsonProperty + public boolean isIgnoreHostname() { + return ignoreHostname; + } + + @JsonProperty + public boolean isIgnoreServiceName() { + return ignoreServiceName; + } + + @JsonProperty + public String getNamespacePrefix() { + return namespacePrefix; + } + + public ImmutableSortedMap> getWhiteListDimsMapper() { + return whiteListDimsMapper; + } + + /** + * @param event Event subject to filtering + * @return true if and only if the event prefix key is in the {@code whiteListDimsMapper} + */ + private boolean isInWhiteList(ServiceMetricEvent event) { + return getPrefixKey(event.getMetric(), whiteListDimsMapper) != null; + } + + /** + * @param key the metric name to lookup + * @param whiteList + * @return null if the key does not match with any of the prefixes keys in @code metricsWhiteList, + * or the prefix in @code whiteListDimsMapper + */ + private String getPrefixKey(String key, SortedMap whiteList) { + String prefixKey = null; + if (whiteList.containsKey(key)) { + return key; + } + SortedMap headMap = whiteList.headMap(key); + if (!headMap.isEmpty() && key.startsWith(headMap.lastKey())) { + prefixKey = headMap.lastKey(); + } + return prefixKey; + } + + /** + * Returns a {@link java.util.List} of the white-listed dimension's values to send. + * The list is order is the same as the order of dimensions {@code whiteListDimsMapper} + * + * @param event the event for which will filter dimensions + * @return {@link java.util.List} of the filtered dimension values to send or null if the event is not in the white list + */ + private List getOrderedDimValues(ServiceMetricEvent event) { + String prefixKey = getPrefixKey(event.getMetric(), whiteListDimsMapper); + if (prefixKey == null) { + return null; + } + ImmutableList.Builder outputList = new ImmutableList.Builder(); + Set dimensions = whiteListDimsMapper.get(prefixKey); + if (dimensions == null) { + return Collections.emptyList(); + } + for (String dimKey : dimensions) { + String dimValue = (String) event.getUserDims().get(dimKey); + if (dimValue != null) { + outputList.add(DropwizardEmitter.sanitize(dimValue)); + } + } + return outputList.build(); + } + + /** + * @param serviceMetricEvent druid metric event to convert + * @return null if the event is not white listed, otherwise return {@link io.druid.emitter.dropwizard.DropwizardEvent} + *

+ * The metric name of the dropwizard event is: + * .[].[].. + *

+ * The order of the dimension is the order returned by {@code getOrderedDimValues()} + * Note that this path will be sanitized by replacing all the `.` or space by `_` {@link io.druid.emitter.dropwizard.DropwizardEmitter#sanitize(String)} + *

+ */ + + @Override + public DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent) { + if (!this.isInWhiteList(serviceMetricEvent)) { + return null; + } + final ImmutableList.Builder metricPathBuilder = new ImmutableList.Builder<>(); + if (!Strings.isNullOrEmpty(this.getNamespacePrefix())) + metricPathBuilder.add(this.getNamespacePrefix()); + if (!this.isIgnoreServiceName()) { + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getService())); + } + if (!this.isIgnoreHostname()) { + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getHost())); + } + metricPathBuilder.addAll(this.getOrderedDimValues(serviceMetricEvent)); + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getMetric())); + + final DropwizardEvent dropwizardEvent = new DropwizardEvent( + Joiner.on(".").join(metricPathBuilder.build()), + serviceMetricEvent.getValue().doubleValue() + ); + return dropwizardEvent; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WhiteListBasedConverter)) { + return false; + } + + WhiteListBasedConverter that = (WhiteListBasedConverter) o; + + if (isIgnoreHostname() != that.isIgnoreHostname()) { + return false; + } + if (isIgnoreServiceName() != that.isIgnoreServiceName()) { + return false; + } + if (!getNamespacePrefix().equals(that.getNamespacePrefix())) { + return false; + } + return mapPath != null ? mapPath.equals(that.mapPath) : that.mapPath == null; + + } + + @Override + public int hashCode() { + int result = (isIgnoreHostname() ? 1 : 0); + result = 31 * result + (isIgnoreServiceName() ? 1 : 0); + result = 31 * result + (Strings.isNullOrEmpty(getNamespacePrefix())?0:getNamespacePrefix().hashCode()); + result = 31 * result + (mapPath != null ? mapPath.hashCode() : 0); + return result; + } + + private ImmutableSortedMap> readMap(final String mapPath) { + String fileContent; + String actualPath = mapPath; + try { + if (Strings.isNullOrEmpty(mapPath)) { + URL resource = this.getClass().getClassLoader().getResource("defaultWhiteListMap.json"); + actualPath = resource.getFile(); + LOGGER.info("using default whiteList map located at [%s]", actualPath); + fileContent = Resources.toString(resource, Charset.defaultCharset()); + } else { + fileContent = Files.asCharSource(new File(mapPath), Charset.forName("UTF-8")).read(); + } + return mapper.reader(new TypeReference>>() { + }).readValue(fileContent); + } catch (IOException e) { + throw new ISE(e, "Got an exception while parsing file [%s]", actualPath); + } + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java new file mode 100644 index 000000000000..6f12cb8b1898 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java @@ -0,0 +1,120 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard.reporters; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.druid.emitter.dropwizard.DropwizardReporter; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@JsonTypeName("console") +public class DropwizardConsoleReporter implements DropwizardReporter { + private long emitIntervalInSecs; + private TimeUnit rates = TimeUnit.SECONDS; + private TimeUnit durations = TimeUnit.MILLISECONDS; + private ConsoleReporter consoleReporter; + + @JsonProperty + public long getEmitIntervalInSecs() { + return emitIntervalInSecs; + } + + @JsonProperty + public void setEmitIntervalInSecs(long emitIntervalInSecs) { + this.emitIntervalInSecs = emitIntervalInSecs; + } + + @JsonProperty + public TimeUnit getRates() { + return rates; + } + + @JsonProperty + public void setRates(String rates) { + this.rates = TimeUnit.valueOf(rates); + } + + @JsonProperty + public TimeUnit getDurations() { + return durations; + } + + @JsonProperty + public void setDurations(String durations) { + this.durations = TimeUnit.valueOf(durations); + } + + @Override + public void start(MetricRegistry metricRegistry) throws IOException { + consoleReporter = com.codahale.metrics.ConsoleReporter.forRegistry(metricRegistry). + convertDurationsTo(durations).convertRatesTo(rates).build(); + consoleReporter.start(emitIntervalInSecs, TimeUnit.SECONDS); + + } + + @Override + public void flush() throws IOException { + } + + @Override + public void close() throws IOException { + + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DropwizardConsoleReporter that = (DropwizardConsoleReporter) o; + + if (emitIntervalInSecs != that.emitIntervalInSecs) { + return false; + } + if (consoleReporter != null ? !consoleReporter.equals(that.consoleReporter) : that.consoleReporter != null) { + return false; + } + if (durations != that.durations) { + return false; + } + if (rates != that.rates) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + int result = (int) (emitIntervalInSecs ^ (emitIntervalInSecs >>> 32)); + result = 31 * result + (rates != null ? rates.hashCode() : 0); + result = 31 * result + (durations != null ? durations.hashCode() : 0); + result = 31 * result + (consoleReporter != null ? consoleReporter.hashCode() : 0); + return result; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java new file mode 100644 index 000000000000..790cf77954a4 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java @@ -0,0 +1,46 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard.reporters; + +import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.druid.emitter.dropwizard.DropwizardReporter; + +import java.io.IOException; + +@JsonTypeName("jmx") +public class DropwizardJMXReporter implements DropwizardReporter { + @Override + public void start(MetricRegistry metricRegistry) throws IOException { + final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry).build(); + reporter.start(); + + } + + @Override + public void flush() throws IOException { + + } + + @Override + public void close() throws IOException { + + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule new file mode 100644 index 000000000000..8c325c3daa89 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule @@ -0,0 +1 @@ +io.druid.emitter.dropwizard.DropwizardEmitterModule diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json new file mode 100644 index 000000000000..87cbd8951653 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json @@ -0,0 +1,48 @@ +{ + "ingest/events": [], + "ingest/handoff/failed": [], + "ingest/persists": [], + "ingest/rows/output": [], + "jvm/gc": [], + "jvm/mem": [], + "query/cpu/time": [ + "dataSource", + "type" + ], + "query/node/time": [ + "dataSource", + "type" + ], + "query/node/ttfb": [ + "dataSource", + "type" + ], + "query/partial/time": [ + "dataSource", + "type" + ], + "query/segment/time": [ + "dataSource", + "type" + ], + "query/segmentAndCache/time": [ + "dataSource", + "type" + ], + "query/time": [ + "dataSource", + "type" + ], + "query/wait/time": [ + "dataSource", + "type" + ], + "segment/count": [], + "segment/dropQueue/count": [], + "segment/loadQueue/count": [], + "segment/loadQueue/failed": [], + "segment/loadQueue/size": [], + "segment/scan/pending": [], + "segment/size": [], + "segment/usedPercent": [] +} diff --git a/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java b/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java new file mode 100644 index 000000000000..b3cf659176af --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java @@ -0,0 +1,63 @@ +package io.druid.emitter.dropwizard; + +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.emitter.dropwizard.eventconverters.SendAllDropwizardEventConverter; +import io.druid.emitter.dropwizard.eventconverters.WhiteListBasedConverter; +import io.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; +import io.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +public class DropwizardEmitterConfigTest { + private ObjectMapper mapper = new DefaultObjectMapper(); + + @Before + public void setUp() + { + mapper.setInjectableValues(new InjectableValues.Std().addValue( + ObjectMapper.class, + new DefaultObjectMapper() + )); + } + + @Test + public void testSerDeserDropwizardEmitterConfig() throws IOException + { + DropwizardEmitterConfig dropwizardEmitterConfig = new DropwizardEmitterConfig(new DropwizardConsoleReporter(),new SendAllDropwizardEventConverter(null,false,false),new HistogramMetricManager(),null); + String dropwizardEmitterConfigString = mapper.writeValueAsString(dropwizardEmitterConfig); + DropwizardEmitterConfig dropwizardEmitterConfigExpected = mapper.reader(DropwizardEmitterConfig.class).readValue( + dropwizardEmitterConfigString + ); + Assert.assertEquals(dropwizardEmitterConfigExpected, dropwizardEmitterConfig); + } + + @Test + public void testSerDeserDruidToDropwizardEventConverter() throws IOException + { + SendAllDropwizardEventConverter sendAllDropwizardEventConverter = new SendAllDropwizardEventConverter("prefix", true, true); + String noopDropwizardEventConverterString = mapper.writeValueAsString(sendAllDropwizardEventConverter); + DruidToDropwizardEventConverter druidToDropwizardEventConverter = mapper.reader(DruidToDropwizardEventConverter.class) + .readValue(noopDropwizardEventConverterString); + Assert.assertEquals(druidToDropwizardEventConverter, sendAllDropwizardEventConverter); + + WhiteListBasedConverter whiteListBasedConverter = new WhiteListBasedConverter( + "prefix", + true, + true, + "", + new DefaultObjectMapper() + ); + String whiteListBasedConverterString = mapper.writeValueAsString(whiteListBasedConverter); + druidToDropwizardEventConverter = mapper.reader(DruidToDropwizardEventConverter.class) + .readValue(whiteListBasedConverterString); + Assert.assertEquals(druidToDropwizardEventConverter, whiteListBasedConverter); + } + + } + + From c76173199ec0723efb2724e9b932b31e3f0e1eff Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 6 Mar 2019 00:02:03 +0530 Subject: [PATCH 02/16] Refactor and make things work more improvements improve docs refactrings --- distribution/pom.xml | 2 + docs/design/extensions-contrib/dropwizard.md | 106 ++-- extensions-contrib/dropwizard-emitter/pom.xml | 36 +- .../emitter/dropwizard/DropwizardEmitter.java | 108 ---- .../dropwizard/DropwizardEmitterConfig.java | 91 --- .../dropwizard/DropwizardEmitterModule.java | 70 --- .../emitter/dropwizard/DropwizardEvent.java | 38 -- .../dropwizard/DropwizardMetricManager.java | 32 -- .../dropwizard/DropwizardReporter.java | 39 -- .../DruidToDropwizardEventConverter.java | 43 -- .../dropwizard/HistogramMetricManager.java | 46 -- .../SendAllDropwizardEventConverter.java | 132 ----- .../WhiteListBasedConverter.java | 253 -------- .../reporters/DropwizardConsoleReporter.java | 120 ---- .../reporters/DropwizardJMXReporter.java | 46 -- .../dropwizard/DropwizardConverter.java | 97 ++++ .../emitter/dropwizard/DropwizardEmitter.java | 181 ++++++ .../dropwizard/DropwizardEmitterConfig.java | 135 +++++ .../dropwizard/DropwizardEmitterModule.java | 70 +++ .../dropwizard/DropwizardMetricSpec.java | 72 +++ .../dropwizard/DropwizardReporter.java | 43 ++ .../reporters/DropwizardConsoleReporter.java | 134 +++++ .../reporters/DropwizardJMXReporter.java | 78 +++ .../io.druid.initialization.DruidModule | 1 - ...rg.apache.druid.initialization.DruidModule | 16 + .../resources/defaultMetricDimensions.json | 540 ++++++++++++++++++ .../main/resources/defaultWhiteListMap.json | 48 -- .../DropwizardEmitterConfigTest.java | 63 -- .../DropwizardEmitterConfigTest.java | 67 +++ extensions-contrib/graphite-emitter/pom.xml | 34 +- .../indexing/overlord/RemoteTaskRunner.java | 3 + pom.xml | 2 + 32 files changed, 1515 insertions(+), 1231 deletions(-) delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardMetricSpec.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule create mode 100644 extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json delete mode 100644 extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json delete mode 100644 extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java create mode 100644 extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java diff --git a/distribution/pom.xml b/distribution/pom.xml index 76466cf9c876..215cd8d95b12 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -373,6 +373,8 @@ -c org.apache.druid.extensions.contrib:ambari-metrics-emitter -c + org.apache.druid.extensions.contrib:dropwizard-emitter + -c org.apache.druid.extensions.contrib:druid-azure-extensions -c org.apache.druid.extensions.contrib:druid-cassandra-storage diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index 17f248239c47..111f49bd5b0f 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -1,18 +1,37 @@ --- layout: doc_page +title: "Dropwizard metrics emitter" --- + + # Dropwizard Emitter To use this extension, make sure to [include](../../operations/including-extensions.html) `dropwizard-emitter` extension. ## Introduction -The intent of this extension is to integrate [Dropwizard](http://metrics.dropwizard.io/3.1.0/getting-started/#) metrics library with druid so that dropwizard users can easily absorb druid into their monitoring ecosystem. -It accumulates druid metrics in a dropwizard histogram and emits them to various sinks via dropwizard supported reporters. -A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, etc., it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles. -Currently dropwizard metrics can be emitted to these sinks: -Console,HTTP,JMX,Graphite,CSV,Slf4jLogger,Ganglia and various other community supported [sinks](http://metrics.dropwizard.io/3.1.0/manual/third-party/). +This extension integrates [Dropwizard](http://metrics.dropwizard.io/3.1.0/getting-started/#) metrics library with druid so that dropwizard users can easily absorb druid into their monitoring ecosystem. +It accumulates druid metrics as dropwizard metrics, and emits them to various sinks via dropwizard supported reporters. +Currently supported dropwizard metrics types counter, gauge, meter, timer and histogram. +These metrics can be emitted using either Console or JMX reporter. ## Configuration @@ -20,79 +39,46 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter |property|description|required?|default| |--------|-----------|---------|-------| -|`druid.emitter.dropwizard.metric`|The metric manager to be used.Currently supported metric manager is histogram|no|histogram| -|`druid.emitter.dropwizard.reporter`|The dropwizard reporter to be used.|yes|none| -|`druid.emitter.dropwizard.eventConverter`| Filter and converter of druid events to dropwizard event(please see next section). |yes|none| +|`druid.emitter.dropwizard.reporters`|List of dropwizard reporters to be used.|yes|none| +|`druid.emitter.dropwizard.prefix`|Optional prefix to be used for metrics name|no|none| +|`druid.emitter.dropwizard.includeHost`|Flag to include the hostname as part of the metric name.|no|yes| +|`druid.emitter.dropwizard.includeDimensionNames`|Flag to include the dimension names as part of the metric name.|no|yes| +|`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the StatsD type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| |`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| -### Druid to Dropwizard Event Converter - -Dropwizard Event Converter defines a mapping between druid metrics name plus dimensions to a Dropwizard metric name. -Dropwizard metric name is organized using the following schema: -`.[].[]..` -Properly naming the metrics is critical to avoid conflicts, confusing data and potentially wrong interpretation later on. - -Example `druid.historical.abc_com:8080.MyDataSourceName.GroupBy.query/time`: - - * `druid` -> namespace prefix - * `historical` -> service name - * `abc.com:8080` -> druid hostname - * `MyDataSourceName` -> dimension value - * `GroupBy` -> dimension value - * `query/time` -> metric name - -We have two different implementation of event converter: - -#### Send-All converter - -The first implementation called `all`, will send all the druid service metrics events. -The metric name will be in the form `.[].[]..` -User has control of `.[].[].` - -You can omit the hostname by setting `ignoreHostname=true` -`druid.SERVICE_NAME.dataSourceName.queryType.query.time` - -You can omit the service name by setting `ignoreServiceName=true` -`druid.HOSTNAME.dataSourceName.queryType.query.time` - -```json +### Druid to Dropwizard Event Conversion -druid.emitter.dropwizard.eventConverter={"type":"all", "namespacePrefix": "druid.test", "ignoreHostname":true, "ignoreServiceName":true} +Each metric emitted using Dropwizard must specify a type, one of `[timer, counter, guage, meter, histogram, ]`. Dropwizard Emitter expects this mapping to +be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric. +If the user does not specify their own JSON file, a default mapping is used. +All metrics are expected to be mapped. Metrics which are not mapped will log an error. -``` +Dropwizard metric path is organized using the following schema: -#### White-list based converter +` : { "dimensions" : , "type" : , "timeUnit" : }` -The second implementation called `whiteList`, will send only the white listed metrics and dimensions. -Same as for the `all` converter user has control of `.[].[].` -White-list based converter comes with the following default white list map located under resources in `./src/main/resources/defaultWhiteListMap.json` +e.g. -Although user can override the default white list map by supplying a property called `mapPath`. -This property is a String containing the path for the file containing **white list map Json object**. -For example the following converter will read the map from the file `/pathPrefix/fileName.json`. +`"query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer", "timeUnit": "MILLISECONDS"}` +`"segment/scan/pending" : { "dimensions" : [], "type" : "gauge"}` -```json -druid.emitter.dropwizard.eventConverter={"type":"whiteList", "namespacePrefix": "druid.test", "ignoreHostname":true, "ignoreServiceName":true, "mapPath":"/pathPrefix/fileName.json"} - -``` +For most use-cases, the default mapping is sufficient. -**Druid emits a huge number of metrics we highly recommend to use the `whiteList` converter** - -### Metric Manager - -Metric manager defines the dropwizard accumulator that would be used to accumulate druid metric events. -For eg : Gauge,Counter,Histogram,Meter etc. - -### Dropwizard reporter +### Supported Dropwizard reporters +#### JMX Reporter +Used to report druid metrics via JMX. ```json druid.emitter.dropwizard.reporter={"type":"jmx"} ``` +#### Console Reporter +Used to print Druid Metrics to console logs. + ```json druid.emitter.dropwizard.reporter={"type":"console","emitIntervalInSecs":30}"} diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml index e0e1ebd3944a..03c655a5b5aa 100644 --- a/extensions-contrib/dropwizard-emitter/pom.xml +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -18,39 +18,39 @@ ~ under the License. --> - 4.0.0 - io.druid + org.apache.druid druid - 0.10.0-SNAPSHOT + 0.15.0-incubating-SNAPSHOT ../../pom.xml - io.druid.extensions.contrib + org.apache.druid.extensions.contrib dropwizard-emitter dropwizard-emitter Druid emitter extension to convert druid metric to Dropwizard metrics - io.druid - druid-common + org.apache.druid + druid-core ${project.parent.version} provided - io.druid - druid-api + org.apache.druid + druid-server ${project.parent.version} provided - com.metamx - emitter - provided + io.dropwizard.metrics + metrics-core + 3.1.0 junit @@ -69,23 +69,11 @@ test - io.druid - druid-server - ${project.parent.version} - test-jar - test - - - io.druid + org.apache.druid druid-processing ${project.parent.version} test-jar test - - io.dropwizard.metrics - metrics-core - 3.1.0 - diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java deleted file mode 100644 index 153c03bbf071..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard; - -import com.codahale.metrics.MetricRegistry; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.service.AlertEvent; -import com.metamx.emitter.service.ServiceMetricEvent; -import io.druid.java.util.common.ISE; -import io.druid.java.util.common.logger.Logger; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.regex.Pattern; - -public class DropwizardEmitter implements Emitter { - private static Logger log = new Logger(DropwizardEmitter.class); - private final MetricRegistry metricsRegistry = new MetricRegistry(); - private final DropwizardMetricManager dropwizardMetricManager; - private final AtomicBoolean started = new AtomicBoolean(false); - private final DruidToDropwizardEventConverter druidToDropwizardEventConverter; - private final List emitterList; - //TODO make this a list of reporters - private final DropwizardReporter dropwizardReporter; - - public DropwizardEmitter(DropwizardEmitterConfig dropwizardEmitterConfig,List emitterList) { - this.dropwizardMetricManager = dropwizardEmitterConfig.getDropwizardMetricManager(); - this.druidToDropwizardEventConverter = dropwizardEmitterConfig.getDruidToDropwizardEventConverter(); - this.emitterList = emitterList; - this.dropwizardReporter = dropwizardEmitterConfig.getDropwizardReporter(); - } - - @Override - public void start() { - try { - startReporters(); - } catch (IOException e) { - log.error(e,"Error while starting Dropwizard reporters"); - } - started.set(true); - } - - @Override - public void emit(Event event) { - if (!started.get()) { - throw new ISE("Emit was called while emitter is yet not initialized"); - } - if (event instanceof ServiceMetricEvent) { - DropwizardEvent dropwizardEvent = druidToDropwizardEventConverter.druidEventToDropwizard((ServiceMetricEvent) event); - if(dropwizardEvent!=null) { - dropwizardMetricManager.updateMetric(metricsRegistry,dropwizardEvent); - }else{ - log.debug("Dropping the service event "+event); - return; - } - } else if (!emitterList.isEmpty() && event instanceof AlertEvent) { - for (Emitter emitter : emitterList) { - emitter.emit(event); - } - } else if (event instanceof AlertEvent) { - AlertEvent alertEvent = (AlertEvent) event; - log.error( - "The following alert is dropped, description is [%s], severity is [%s]", - alertEvent.getDescription(), alertEvent.getSeverity() - ); - } else { - log.error("unknown event type [%s]", event.getClass()); - } - } - - @Override - public void flush() throws IOException { - dropwizardReporter.flush(); - } - - @Override - public void close() throws IOException { - dropwizardReporter.close(); - } - - private void startReporters() throws IOException { - dropwizardReporter.start(metricsRegistry); - } - - public static String sanitize(String namespace) - { - Pattern DOT_OR_WHITESPACE = Pattern.compile("[\\s]+|[.]+"); - return DOT_OR_WHITESPACE.matcher(namespace).replaceAll("_"); - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java deleted file mode 100644 index 5a642cc18689..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Collections; -import java.util.List; - - -public class DropwizardEmitterConfig { - @JsonProperty("eventConverter") - final private DruidToDropwizardEventConverter druidToDropwizardEventConverter; - @JsonProperty("reporter") - final private DropwizardReporter dropwizardReporter; - @JsonProperty("metric") - final private DropwizardMetricManager dropwizardMetricManager; - @JsonProperty - final private List alertEmitters; - - @JsonCreator - public DropwizardEmitterConfig(@JsonProperty("reporter") DropwizardReporter dropwizardReporter ,@JsonProperty("eventConverter") DruidToDropwizardEventConverter druidToDropwizardEventConverter,@JsonProperty("metric") DropwizardMetricManager dropwizardMetricManager,@JsonProperty("alertEmitters") List alertEmitters ) { - this.dropwizardReporter = dropwizardReporter; - this.druidToDropwizardEventConverter = druidToDropwizardEventConverter; - this.dropwizardMetricManager = dropwizardMetricManager==null? new HistogramMetricManager():dropwizardMetricManager; - this.alertEmitters = alertEmitters == null ? Collections.emptyList() : alertEmitters; - } - - - public DruidToDropwizardEventConverter getDruidToDropwizardEventConverter() { - return druidToDropwizardEventConverter; - } - - public DropwizardReporter getDropwizardReporter() { - return dropwizardReporter; - } - - public List getAlertEmitters() { - return alertEmitters; - } - - public DropwizardMetricManager getDropwizardMetricManager() { - return dropwizardMetricManager; - } - - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - DropwizardEmitterConfig that = (DropwizardEmitterConfig) o; - - if (alertEmitters != null ? !alertEmitters.equals(that.alertEmitters) : that.alertEmitters != null) - return false; - if (dropwizardMetricManager != null ? !dropwizardMetricManager.equals(that.dropwizardMetricManager) : that.dropwizardMetricManager != null) - return false; - if (dropwizardReporter != null ? !dropwizardReporter.equals(that.dropwizardReporter) : that.dropwizardReporter != null) - return false; - if (druidToDropwizardEventConverter != null ? !druidToDropwizardEventConverter.equals(that.druidToDropwizardEventConverter) : that.druidToDropwizardEventConverter != null) - return false; - - return true; - } - - @Override - public int hashCode() { - int result = druidToDropwizardEventConverter != null ? druidToDropwizardEventConverter.hashCode() : 0; - result = 31 * result + (dropwizardReporter != null ? dropwizardReporter.hashCode() : 0); - result = 31 * result + (dropwizardMetricManager != null ? dropwizardMetricManager.hashCode() : 0); - result = 31 * result + (alertEmitters != null ? alertEmitters.hashCode() : 0); - return result; - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java deleted file mode 100644 index 769acc3e0b9b..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard; - -import com.fasterxml.jackson.databind.Module; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Function; -import com.google.common.collect.Lists; -import com.google.inject.Binder; -import com.google.inject.Injector; -import com.google.inject.Key; -import com.google.inject.Provides; -import com.google.inject.name.Named; -import com.google.inject.name.Names; -import com.metamx.emitter.core.Emitter; -import io.druid.guice.JsonConfigProvider; -import io.druid.guice.ManageLifecycle; -import io.druid.initialization.DruidModule; - -import java.util.Collections; -import java.util.List; - -public class DropwizardEmitterModule implements DruidModule -{ - private static final String EMITTER_TYPE = "dropwizard"; - @Override - public List getJacksonModules() - { - return Collections.EMPTY_LIST; - } - - @Override - public void configure(Binder binder) - { - JsonConfigProvider.bind(binder, "druid.emitter." + EMITTER_TYPE, DropwizardEmitterConfig.class); - - } - - @Provides - @ManageLifecycle - @Named(EMITTER_TYPE) - public Emitter getEmitter(DropwizardEmitterConfig dropwizardEmitterConfig, ObjectMapper mapper, final Injector injector){ - List emitters = Lists.transform( - dropwizardEmitterConfig.getAlertEmitters(), - new Function() { - @Override - public Emitter apply(String s) { - return injector.getInstance(Key.get(Emitter.class, Names.named(s))); - } - } - ); - return new DropwizardEmitter(dropwizardEmitterConfig, emitters); - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java deleted file mode 100644 index c32e9a8ab431..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard; - - -public class DropwizardEvent { - private final String metricName; - private final Double value; - - public DropwizardEvent(String metricName, Double value) { - this.metricName = metricName; - this.value = value; - } - - public String getMetricName() { - return metricName; - } - - public Double getValue() { - return value; - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java deleted file mode 100644 index d7a7fcf92210..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard; - -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = HistogramMetricManager.class) -@JsonSubTypes(value= { - @JsonSubTypes.Type(name="histogram", value = HistogramMetricManager.class) -}) - -public interface DropwizardMetricManager { - public void updateMetric(MetricRegistry metricsRegistry,DropwizardEvent dropwizardEvent); -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java deleted file mode 100644 index b2f39a65428f..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard; - -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; -import io.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; - -import java.io.IOException; - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -@JsonSubTypes(value= { - @JsonSubTypes.Type(name="console", value = DropwizardConsoleReporter.class), - @JsonSubTypes.Type(name="jmx", value = DropwizardJMXReporter.class), -}) -public interface DropwizardReporter { - - public void start(MetricRegistry metricRegistry) throws IOException; - public void flush() throws IOException; - public void close() throws IOException; -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java deleted file mode 100644 index 9042acb7bb9b..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.emitter.service.ServiceMetricEvent; -import io.druid.emitter.dropwizard.eventconverters.SendAllDropwizardEventConverter; -import io.druid.emitter.dropwizard.eventconverters.WhiteListBasedConverter; - - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = SendAllDropwizardEventConverter.class) -@JsonSubTypes(value= { - @JsonSubTypes.Type(name="all", value = SendAllDropwizardEventConverter.class), - @JsonSubTypes.Type(name="whiteList", value = WhiteListBasedConverter.class) -}) -public interface DruidToDropwizardEventConverter { - /** - * This function acts as a filter. It returns null if the event is not suppose to be emitted to Dropwizard - * Also This function will define the conversion between the druid event dimension's values and Dropwizard's metric name - * - * @param serviceMetricEvent Druid event ot type {@link com.metamx.emitter.service.ServiceMetricEvent} - * - * @return {@link io.druid.emitter.dropwizard.DropwizardEvent} or null - */ - DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent); -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java deleted file mode 100644 index 4ea72b5ce886..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard; - -import com.codahale.metrics.MetricRegistry; -import io.druid.java.util.common.logger.Logger; -import io.druid.math.expr.Expr; - - -public class HistogramMetricManager implements DropwizardMetricManager { - private static Logger log = new Logger(HistogramMetricManager.class); - @Override - public void updateMetric(MetricRegistry metricsRegistry,DropwizardEvent dropwizardEvent) { - if(dropwizardEvent==null){ - log.error("Dropwizard event passed to " + getClass() + " is null"); - return; - } - metricsRegistry.histogram(dropwizardEvent.getMetricName()).update(dropwizardEvent.getValue().longValue()); - } - - @Override - public boolean equals(Object o){ - return true; - } - - @Override - public int hashCode() { - return 1; - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java deleted file mode 100644 index dce204f15a7e..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard.eventconverters; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSortedSet; -import com.metamx.emitter.service.ServiceMetricEvent; -import io.druid.emitter.dropwizard.DropwizardEmitter; -import io.druid.emitter.dropwizard.DropwizardEvent; -import io.druid.emitter.dropwizard.DruidToDropwizardEventConverter; - - -public class SendAllDropwizardEventConverter implements DruidToDropwizardEventConverter { - @JsonProperty - private boolean ignoreHostname; - @JsonProperty - private boolean ignoreServiceName; - @JsonProperty - private final String namespacePrefix; - - public String getNamespacePrefix() { - return namespacePrefix; - } - - @JsonCreator - public SendAllDropwizardEventConverter( - @JsonProperty("namespacePrefix") String namespacePrefix, - @JsonProperty("ignoreHostname") Boolean ignoreHostname, - @JsonProperty("ignoreServiceName") Boolean ignoreServiceName - ) - { - this.ignoreHostname = ignoreHostname == null ? false : ignoreHostname; - this.ignoreServiceName = ignoreServiceName == null ? false : ignoreServiceName; - this.namespacePrefix = namespacePrefix; - } - - @JsonProperty - public void setIgnoreHostname(boolean ignoreHostname) { - this.ignoreHostname = ignoreHostname; - } - - @JsonProperty - public void setIgnoreServiceName(boolean ignoreServiceName) { - this.ignoreServiceName = ignoreServiceName; - } - - @JsonProperty - public boolean isIgnoreServiceName() - { - return ignoreServiceName; - } - - @JsonProperty - public boolean isIgnoreHostname() - { - return ignoreHostname; - } - - @Override - public DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent) { - ImmutableList.Builder metricPathBuilder = new ImmutableList.Builder(); - - if (!Strings.isNullOrEmpty(this.getNamespacePrefix())) - metricPathBuilder.add(this.getNamespacePrefix()); - - if (!this.isIgnoreServiceName()) { - metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getService())); - } - if (!this.isIgnoreHostname()) { - metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getHost())); - } - - ImmutableSortedSet dimNames = ImmutableSortedSet.copyOf(serviceMetricEvent.getUserDims().keySet()); - for (String dimName : dimNames) { - metricPathBuilder.add(DropwizardEmitter.sanitize(String.valueOf(serviceMetricEvent.getUserDims() - .get(dimName)))); - } - metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getMetric())); - - return new DropwizardEvent(Joiner.on(".").join(metricPathBuilder.build()), - serviceMetricEvent.getValue().doubleValue()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()){ - return false; - } - - SendAllDropwizardEventConverter that = (SendAllDropwizardEventConverter) o; - - if (ignoreHostname != that.ignoreHostname) { - return false; - } - if (ignoreServiceName != that.ignoreServiceName) { - return false; - } - - return true; - } - - @Override - public int hashCode() { - int result = (ignoreHostname ? 1 : 0); - result = 31 * result + (ignoreServiceName ? 1 : 0); - return result; - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java deleted file mode 100644 index 59dfa6d16bb4..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java +++ /dev/null @@ -1,253 +0,0 @@ -/* -* Licensed to Metamarkets Group Inc. (Metamarkets) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. Metamarkets licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ - -package io.druid.emitter.dropwizard.eventconverters; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedMap; -import com.google.common.io.Files; -import com.google.common.io.Resources; -import com.metamx.emitter.service.ServiceMetricEvent; -import io.druid.emitter.dropwizard.DropwizardEmitter; -import io.druid.emitter.dropwizard.DropwizardEvent; -import io.druid.emitter.dropwizard.DruidToDropwizardEventConverter; -import io.druid.java.util.common.ISE; -import io.druid.java.util.common.logger.Logger; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.nio.charset.Charset; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.TimeUnit; - -@JsonTypeName("whiteList") -public class WhiteListBasedConverter implements DruidToDropwizardEventConverter { - private static final Logger LOGGER = new Logger(WhiteListBasedConverter.class); - /** - * @code whiteListDimsMapper is a white list of metric->dimensions mappings. - * Key is the metric name or the metric's prefix. - * Value is a list of metric's dimensions names. - * The order of the dimension name is important, it will be used to build the dropwizard metric nanme. - * For instance we have dimension type is nested under dimension dataSource -> prefix.dataSource.queryType.metricName - */ - private final ImmutableSortedMap> whiteListDimsMapper; - - @JsonProperty - private final boolean ignoreHostname; - - @JsonProperty - private final boolean ignoreServiceName; - - @JsonProperty - private final String namespacePrefix; - - @JsonProperty - private final String mapPath; - - private final ObjectMapper mapper; - - @JsonCreator - public WhiteListBasedConverter( - @JsonProperty("namespacePrefix") String namespacePrefix, - @JsonProperty("ignoreHostname") Boolean ignoreHostname, - @JsonProperty("ignoreServiceName") Boolean ignoreServiceName, - @JsonProperty("mapPath") String mapPath, - @JacksonInject ObjectMapper mapper - ) { - this.mapper = Preconditions.checkNotNull(mapper); - this.mapPath = mapPath; - this.whiteListDimsMapper = readMap(this.mapPath); - this.ignoreHostname = ignoreHostname == null ? false : ignoreHostname; - this.ignoreServiceName = ignoreServiceName == null ? false : ignoreServiceName; - this.namespacePrefix = namespacePrefix; - } - - @JsonProperty - public boolean isIgnoreHostname() { - return ignoreHostname; - } - - @JsonProperty - public boolean isIgnoreServiceName() { - return ignoreServiceName; - } - - @JsonProperty - public String getNamespacePrefix() { - return namespacePrefix; - } - - public ImmutableSortedMap> getWhiteListDimsMapper() { - return whiteListDimsMapper; - } - - /** - * @param event Event subject to filtering - * @return true if and only if the event prefix key is in the {@code whiteListDimsMapper} - */ - private boolean isInWhiteList(ServiceMetricEvent event) { - return getPrefixKey(event.getMetric(), whiteListDimsMapper) != null; - } - - /** - * @param key the metric name to lookup - * @param whiteList - * @return null if the key does not match with any of the prefixes keys in @code metricsWhiteList, - * or the prefix in @code whiteListDimsMapper - */ - private String getPrefixKey(String key, SortedMap whiteList) { - String prefixKey = null; - if (whiteList.containsKey(key)) { - return key; - } - SortedMap headMap = whiteList.headMap(key); - if (!headMap.isEmpty() && key.startsWith(headMap.lastKey())) { - prefixKey = headMap.lastKey(); - } - return prefixKey; - } - - /** - * Returns a {@link java.util.List} of the white-listed dimension's values to send. - * The list is order is the same as the order of dimensions {@code whiteListDimsMapper} - * - * @param event the event for which will filter dimensions - * @return {@link java.util.List} of the filtered dimension values to send or null if the event is not in the white list - */ - private List getOrderedDimValues(ServiceMetricEvent event) { - String prefixKey = getPrefixKey(event.getMetric(), whiteListDimsMapper); - if (prefixKey == null) { - return null; - } - ImmutableList.Builder outputList = new ImmutableList.Builder(); - Set dimensions = whiteListDimsMapper.get(prefixKey); - if (dimensions == null) { - return Collections.emptyList(); - } - for (String dimKey : dimensions) { - String dimValue = (String) event.getUserDims().get(dimKey); - if (dimValue != null) { - outputList.add(DropwizardEmitter.sanitize(dimValue)); - } - } - return outputList.build(); - } - - /** - * @param serviceMetricEvent druid metric event to convert - * @return null if the event is not white listed, otherwise return {@link io.druid.emitter.dropwizard.DropwizardEvent} - *

- * The metric name of the dropwizard event is: - * .[].[].. - *

- * The order of the dimension is the order returned by {@code getOrderedDimValues()} - * Note that this path will be sanitized by replacing all the `.` or space by `_` {@link io.druid.emitter.dropwizard.DropwizardEmitter#sanitize(String)} - *

- */ - - @Override - public DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent) { - if (!this.isInWhiteList(serviceMetricEvent)) { - return null; - } - final ImmutableList.Builder metricPathBuilder = new ImmutableList.Builder<>(); - if (!Strings.isNullOrEmpty(this.getNamespacePrefix())) - metricPathBuilder.add(this.getNamespacePrefix()); - if (!this.isIgnoreServiceName()) { - metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getService())); - } - if (!this.isIgnoreHostname()) { - metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getHost())); - } - metricPathBuilder.addAll(this.getOrderedDimValues(serviceMetricEvent)); - metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getMetric())); - - final DropwizardEvent dropwizardEvent = new DropwizardEvent( - Joiner.on(".").join(metricPathBuilder.build()), - serviceMetricEvent.getValue().doubleValue() - ); - return dropwizardEvent; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof WhiteListBasedConverter)) { - return false; - } - - WhiteListBasedConverter that = (WhiteListBasedConverter) o; - - if (isIgnoreHostname() != that.isIgnoreHostname()) { - return false; - } - if (isIgnoreServiceName() != that.isIgnoreServiceName()) { - return false; - } - if (!getNamespacePrefix().equals(that.getNamespacePrefix())) { - return false; - } - return mapPath != null ? mapPath.equals(that.mapPath) : that.mapPath == null; - - } - - @Override - public int hashCode() { - int result = (isIgnoreHostname() ? 1 : 0); - result = 31 * result + (isIgnoreServiceName() ? 1 : 0); - result = 31 * result + (Strings.isNullOrEmpty(getNamespacePrefix())?0:getNamespacePrefix().hashCode()); - result = 31 * result + (mapPath != null ? mapPath.hashCode() : 0); - return result; - } - - private ImmutableSortedMap> readMap(final String mapPath) { - String fileContent; - String actualPath = mapPath; - try { - if (Strings.isNullOrEmpty(mapPath)) { - URL resource = this.getClass().getClassLoader().getResource("defaultWhiteListMap.json"); - actualPath = resource.getFile(); - LOGGER.info("using default whiteList map located at [%s]", actualPath); - fileContent = Resources.toString(resource, Charset.defaultCharset()); - } else { - fileContent = Files.asCharSource(new File(mapPath), Charset.forName("UTF-8")).read(); - } - return mapper.reader(new TypeReference>>() { - }).readValue(fileContent); - } catch (IOException e) { - throw new ISE(e, "Got an exception while parsing file [%s]", actualPath); - } - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java deleted file mode 100644 index 6f12cb8b1898..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard.reporters; - -import com.codahale.metrics.ConsoleReporter; -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.druid.emitter.dropwizard.DropwizardReporter; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -@JsonTypeName("console") -public class DropwizardConsoleReporter implements DropwizardReporter { - private long emitIntervalInSecs; - private TimeUnit rates = TimeUnit.SECONDS; - private TimeUnit durations = TimeUnit.MILLISECONDS; - private ConsoleReporter consoleReporter; - - @JsonProperty - public long getEmitIntervalInSecs() { - return emitIntervalInSecs; - } - - @JsonProperty - public void setEmitIntervalInSecs(long emitIntervalInSecs) { - this.emitIntervalInSecs = emitIntervalInSecs; - } - - @JsonProperty - public TimeUnit getRates() { - return rates; - } - - @JsonProperty - public void setRates(String rates) { - this.rates = TimeUnit.valueOf(rates); - } - - @JsonProperty - public TimeUnit getDurations() { - return durations; - } - - @JsonProperty - public void setDurations(String durations) { - this.durations = TimeUnit.valueOf(durations); - } - - @Override - public void start(MetricRegistry metricRegistry) throws IOException { - consoleReporter = com.codahale.metrics.ConsoleReporter.forRegistry(metricRegistry). - convertDurationsTo(durations).convertRatesTo(rates).build(); - consoleReporter.start(emitIntervalInSecs, TimeUnit.SECONDS); - - } - - @Override - public void flush() throws IOException { - } - - @Override - public void close() throws IOException { - - } - - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - DropwizardConsoleReporter that = (DropwizardConsoleReporter) o; - - if (emitIntervalInSecs != that.emitIntervalInSecs) { - return false; - } - if (consoleReporter != null ? !consoleReporter.equals(that.consoleReporter) : that.consoleReporter != null) { - return false; - } - if (durations != that.durations) { - return false; - } - if (rates != that.rates) { - return false; - } - - return true; - } - - @Override - public int hashCode() { - int result = (int) (emitIntervalInSecs ^ (emitIntervalInSecs >>> 32)); - result = 31 * result + (rates != null ? rates.hashCode() : 0); - result = 31 * result + (durations != null ? durations.hashCode() : 0); - result = 31 * result + (consoleReporter != null ? consoleReporter.hashCode() : 0); - return result; - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java deleted file mode 100644 index 790cf77954a4..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package io.druid.emitter.dropwizard.reporters; - -import com.codahale.metrics.JmxReporter; -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.druid.emitter.dropwizard.DropwizardReporter; - -import java.io.IOException; - -@JsonTypeName("jmx") -public class DropwizardJMXReporter implements DropwizardReporter { - @Override - public void start(MetricRegistry metricRegistry) throws IOException { - final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry).build(); - reporter.start(); - - } - - @Override - public void flush() throws IOException { - - } - - @Override - public void close() throws IOException { - - } -} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java new file mode 100644 index 000000000000..e584486c173b --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.logger.Logger; + +import javax.annotation.Nullable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +/** + * + */ +public class DropwizardConverter +{ + private static final Logger log = new Logger(DropwizardConverter.class); + private Map metricMap; + + public DropwizardConverter(ObjectMapper mapper, String dimensionMapPath) + { + metricMap = readMap(mapper, dimensionMapPath); + } + + @Nullable + public DropwizardMetricSpec addFilteredUserDims( + String service, + String metric, + Map userDims, + Map builder + ) + { + /* + Find the metric in the map. If we cant find it try to look it up prefixed by the service name. + This is because some metrics are reported differently, but with the same name, from different services. + */ + DropwizardMetricSpec metricSpec = null; + if (metricMap.containsKey(metric)) { + metricSpec = metricMap.get(metric); + } else if (metricMap.containsKey(service + "-" + metric)) { + metricSpec = metricMap.get(service + "-" + metric); + } + if (metricSpec != null) { + for (String dim : metricSpec.getDimensions()) { + if (userDims.containsKey(dim)) { + builder.put(dim, userDims.get(dim).toString()); + } + } + return metricSpec; + } else { + return null; + } + } + + private Map readMap(ObjectMapper mapper, String dimensionMapPath) + { + try { + InputStream is; + if (Strings.isNullOrEmpty(dimensionMapPath)) { + log.info("Using default metric dimension and types"); + is = this.getClass().getClassLoader().getResourceAsStream("defaultMetricDimensions.json"); + } else { + log.info("Using metric dimensions at types at [%s]", dimensionMapPath); + is = new FileInputStream(new File(dimensionMapPath)); + } + return mapper.readerFor(new TypeReference>() + { + }).readValue(is); + } + catch (IOException e) { + throw new ISE(e, "Failed to parse metric dimensions and types"); + } + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java new file mode 100644 index 000000000000..c32997e3f41d --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.core.Emitter; +import org.apache.druid.java.util.emitter.core.Event; +import org.apache.druid.java.util.emitter.service.AlertEvent; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +public class DropwizardEmitter implements Emitter +{ + private static Logger log = new Logger(DropwizardEmitter.class); + private final MetricRegistry metricsRegistry = new MetricRegistry(); + private final AtomicBoolean started = new AtomicBoolean(false); + private final DropwizardConverter converter; + private final List alertEmitters; + private final List reporters; + private final DropwizardEmitterConfig config; + // Note: the gauges do not represent the actual instantaneous value for the metrics. + // Instead they have the last known value for it. + private final ConcurrentHashMap gagues = new ConcurrentHashMap<>(); + + public DropwizardEmitter( + DropwizardEmitterConfig config, + ObjectMapper mapper, + List alertEmitters + ) + { + this.alertEmitters = alertEmitters; + this.config = config; + this.reporters = config.getReporters(); + this.converter = new DropwizardConverter(mapper, config.getDimensionMapPath()); + } + + + @Override + public void start() + { + for (DropwizardReporter reporter : reporters) { + try { + reporter.start(metricsRegistry); + } + catch (IOException e) { + log.error(e, "Error while flushing Dropwizard reporter"); + } + } + started.set(true); + } + + @Override + public void emit(Event event) + { + if (event instanceof ServiceMetricEvent) { + ServiceMetricEvent metricEvent = (ServiceMetricEvent) event; + String host = metricEvent.getHost(); + String service = metricEvent.getService(); + String metric = metricEvent.getMetric(); + Map userDims = metricEvent.getUserDims(); + Number value = metricEvent.getValue(); + ImmutableList.Builder nameBuilder = new ImmutableList.Builder<>(); + LinkedHashMap dims = new LinkedHashMap<>(); + final DropwizardMetricSpec metricSpec = converter.addFilteredUserDims(service, metric, userDims, dims); + + if (metricSpec != null) { + if (config.isIncludeDimensionNames()) { + if (config.getPrefix() != null) { + nameBuilder.add(config.getPrefix()); + } + nameBuilder.add("metric=" + metric); + nameBuilder.add("service=" + service); + if (config.getIncludeHost()) { + nameBuilder.add("hostname=" + host); + } + dims.forEach((key, value1) -> nameBuilder.add(key + "=" + value1)); + } else { + nameBuilder.add(metric); + nameBuilder.add(service); + if (config.getIncludeHost()) { + nameBuilder.add(host); + } + nameBuilder.addAll(dims.values()); + } + + String fullName = StringUtils.replaceChar(Joiner.on(",").join(nameBuilder.build()), '/', "."); + updateMetric(fullName, value, metricSpec); + } else { + log.debug("Service=[%s], Metric=[%s] has no mapping", service, metric); + } + } else if (event instanceof AlertEvent) { + for (Emitter emitter : alertEmitters) { + emitter.emit(event); + } + } else { + throw new ISE("unknown event type [%s]", event.getClass()); + } + } + + private void updateMetric(String name, Number value, DropwizardMetricSpec metricSpec) + { + switch (metricSpec.getType()) { + case meter: + metricsRegistry.meter(name).mark(value.longValue()); + break; + case timer: + metricsRegistry.timer(name) + .update(value.longValue(), metricSpec.getTimeUnit()); + break; + case counter: + metricsRegistry.counter(name).inc(value.longValue()); + break; + case histogram: + metricsRegistry.histogram(name).update(value.longValue()); + break; + case gauge: + Number prevVal = gagues.put(name, value); + // It is the first time we get this metric, register it with metricsRegistry + if (prevVal == null) { + metricsRegistry.register(name, (Gauge) () -> gagues.get(name)); + } + } + } + + @Override + public void flush() + { + for (DropwizardReporter reporter : reporters) { + try { + reporter.flush(); + } + catch (IOException e) { + log.error(e, "Error while flushing Dropwizard reporter"); + } + } + } + + @Override + public void close() + { + for (DropwizardReporter reporter : reporters) { + try { + reporter.close(); + } + catch (IOException e) { + log.error(e, "Error while closing Dropwizard reporter"); + } + } + } + +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java new file mode 100644 index 000000000000..6977e4265dcd --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + + +public class DropwizardEmitterConfig +{ + @JsonProperty + private final List reporters; + @JsonProperty + private final String prefix; + @JsonProperty + private final Boolean includeHost; + @JsonProperty + private final String dimensionMapPath; + @JsonProperty + private final Boolean includeDimensionNames; + @JsonProperty + private final List alertEmitters; + + @JsonCreator + public DropwizardEmitterConfig( + @JsonProperty("reporters") List reporters, + @JsonProperty("prefix") String prefix, + @JsonProperty("includeHost") Boolean includeHost, + @JsonProperty("dimensionMapPath") String dimensionMapPath, + @JsonProperty("includeDimensionNames") Boolean includeDimensionNames, + @JsonProperty("alertEmitters") List alertEmitters + ) + { + this.reporters = reporters; + this.prefix = prefix; + this.alertEmitters = alertEmitters == null ? Collections.emptyList() : alertEmitters; + this.includeHost = includeHost != null ? includeHost : true; + this.dimensionMapPath = dimensionMapPath; + this.includeDimensionNames = includeDimensionNames != null ? includeDimensionNames : true; + } + + @JsonProperty + public List getReporters() + { + return reporters; + } + + @JsonProperty + public String getPrefix() + { + return prefix; + } + + @JsonProperty + public Boolean getIncludeHost() + { + return includeHost; + } + + @JsonProperty + public String getDimensionMapPath() + { + return dimensionMapPath; + } + + @JsonProperty + public Boolean isIncludeDimensionNames() + { + return includeDimensionNames; + } + + @JsonProperty + public List getAlertEmitters() + { + return alertEmitters; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DropwizardEmitterConfig that = (DropwizardEmitterConfig) o; + return Objects.equals(reporters, that.reporters) && + Objects.equals(prefix, that.prefix) && + Objects.equals(includeHost, that.includeHost) && + Objects.equals(dimensionMapPath, that.dimensionMapPath) && + Objects.equals(includeDimensionNames, that.includeDimensionNames) && + Objects.equals(alertEmitters, that.alertEmitters); + } + + @Override + public int hashCode() + { + return Objects.hash(reporters, prefix, includeHost, dimensionMapPath, includeDimensionNames, alertEmitters); + } + + @Override + public String toString() + { + return "DropwizardEmitterConfig{" + + "reporters=" + reporters + + ", prefix='" + prefix + '\'' + + ", includeHost=" + includeHost + + ", dimensionMapPath='" + dimensionMapPath + '\'' + + ", includeDimensionNames=" + includeDimensionNames + + ", alertEmitters=" + alertEmitters + + '}'; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java new file mode 100644 index 000000000000..0ad7000f655c --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.java.util.emitter.core.Emitter; + +import java.util.Collections; +import java.util.List; + +public class DropwizardEmitterModule implements DruidModule +{ + private static final String EMITTER_TYPE = "dropwizard"; + + @Override + public List getJacksonModules() + { + return Collections.EMPTY_LIST; + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter." + EMITTER_TYPE, DropwizardEmitterConfig.class); + } + + @Provides + @ManageLifecycle + @Named(EMITTER_TYPE) + public Emitter getEmitter( + DropwizardEmitterConfig dropwizardEmitterConfig, + ObjectMapper mapper, + final Injector injector + ) + { + List alertEmitters = Lists.transform( + dropwizardEmitterConfig.getAlertEmitters(), + s -> injector.getInstance(Key.get(Emitter.class, Names.named(s))) + ); + return new DropwizardEmitter(dropwizardEmitterConfig, mapper, alertEmitters); + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardMetricSpec.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardMetricSpec.java new file mode 100644 index 000000000000..98a652928106 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardMetricSpec.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class DropwizardMetricSpec +{ + @JsonProperty("dimensions") + private final List dimensions; + @JsonProperty("type") + private final Type type; + @JsonProperty("timeUnit") + private final TimeUnit timeUnit; + + @JsonCreator + DropwizardMetricSpec( + @JsonProperty("dimensions") List dimensions, + @JsonProperty("type") Type type, + @JsonProperty("timeUnit") TimeUnit timeUnit + ) + { + this.dimensions = dimensions; + this.type = type; + this.timeUnit = timeUnit; + } + + @JsonProperty + public Type getType() + { + return type; + } + + @JsonProperty + public List getDimensions() + { + return dimensions; + } + + @JsonProperty + public TimeUnit getTimeUnit() + { + return timeUnit; + } + + public enum Type + { + histogram, timer, meter, counter, gauge + } + +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java new file mode 100644 index 000000000000..4bca4c5fc730 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; +import org.apache.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; + +import java.io.IOException; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "console", value = DropwizardConsoleReporter.class), + @JsonSubTypes.Type(name = "jmx", value = DropwizardJMXReporter.class), +}) +public interface DropwizardReporter +{ + + void start(MetricRegistry metricRegistry) throws IOException; + + void flush() throws IOException; + + void close() throws IOException; +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java new file mode 100644 index 000000000000..8b8559bfb0e1 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard.reporters; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.druid.emitter.dropwizard.DropwizardReporter; + +import java.util.concurrent.TimeUnit; + +@JsonTypeName("console") +public class DropwizardConsoleReporter implements DropwizardReporter +{ + private long emitIntervalInSecs; + private TimeUnit rates = TimeUnit.SECONDS; + private TimeUnit durations = TimeUnit.MILLISECONDS; + private ConsoleReporter consoleReporter; + + @JsonProperty + public long getEmitIntervalInSecs() + { + return emitIntervalInSecs; + } + + @JsonProperty + public void setEmitIntervalInSecs(long emitIntervalInSecs) + { + this.emitIntervalInSecs = emitIntervalInSecs; + } + + @JsonProperty + public TimeUnit getRates() + { + return rates; + } + + @JsonProperty + public void setRates(String rates) + { + this.rates = TimeUnit.valueOf(rates); + } + + @JsonProperty + public TimeUnit getDurations() + { + return durations; + } + + @JsonProperty + public void setDurations(String durations) + { + this.durations = TimeUnit.valueOf(durations); + } + + @Override + public void start(MetricRegistry metricRegistry) + { + consoleReporter = ConsoleReporter.forRegistry(metricRegistry) + .convertDurationsTo(durations) + .convertRatesTo(rates) + .build(); + consoleReporter.start(emitIntervalInSecs, TimeUnit.SECONDS); + + } + + @Override + public void flush() + { + } + + @Override + public void close() + { + + } + + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DropwizardConsoleReporter that = (DropwizardConsoleReporter) o; + + if (emitIntervalInSecs != that.emitIntervalInSecs) { + return false; + } + if (consoleReporter != null ? !consoleReporter.equals(that.consoleReporter) : that.consoleReporter != null) { + return false; + } + if (durations != that.durations) { + return false; + } + if (rates != that.rates) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = (int) (emitIntervalInSecs ^ (emitIntervalInSecs >>> 32)); + result = 31 * result + (rates != null ? rates.hashCode() : 0); + result = 31 * result + (durations != null ? durations.hashCode() : 0); + result = 31 * result + (consoleReporter != null ? consoleReporter.hashCode() : 0); + return result; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java new file mode 100644 index 000000000000..dcbb8027a94e --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard.reporters; + +import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.druid.emitter.dropwizard.DropwizardReporter; + +import java.util.Objects; + +@JsonTypeName("jmx") +public class DropwizardJMXReporter implements DropwizardReporter +{ + private String domain = "org.apache.druid"; + + @JsonProperty + public String getDomain() + { + return domain; + } + + @Override + public void start(MetricRegistry metricRegistry) + { + final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry) + .inDomain(domain).build(); + reporter.start(); + } + + @Override + public void flush() + { + } + + @Override + public void close() + { + + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DropwizardJMXReporter that = (DropwizardJMXReporter) o; + return Objects.equals(domain, that.domain); + } + + @Override + public int hashCode() + { + return Objects.hash(domain); + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule deleted file mode 100644 index 8c325c3daa89..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule +++ /dev/null @@ -1 +0,0 @@ -io.druid.emitter.dropwizard.DropwizardEmitterModule diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..c6d84260ee30 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.emitter.dropwizard.DropwizardEmitterModule diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json new file mode 100644 index 000000000000..ed071968a4f2 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json @@ -0,0 +1,540 @@ +{ + "query/time": { + "dimensions": [ + "dataSource", + "type" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/time": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/ttfb": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/backpressure": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/intervalChunk/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/segment/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/wait/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "segment/scan/pending": { + "dimensions": [], + "type": "gauge" + }, + "query/segmentAndCache/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/cpu/time": { + "dimensions": [ + "dataSource", + "type" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "query/cache/delta/numEntries": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/sizeBytes": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/hits": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/misses": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/evictions": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/hitRate": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/averageBytes": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/timeouts": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/errors": { + "dimensions": [], + "type": "counter" + }, + "query/cache/total/numEntries": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/sizeBytes": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/hits": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/misses": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/evictions": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/hitRate": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/averageBytes": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/timeouts": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/errors": { + "dimensions": [], + "type": "gauge" + }, + "ingest/events/thrownAway": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/unparseable": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/duplicate": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/processed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/rows/output": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/persist/counter": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/persist/time": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "ingest/persist/cpu": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "ingest/persist/backPressure": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "ingest/persist/failed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/handoff/failed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/merge/time": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "ingest/merge/cpu": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "task/run/time": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "segment/added/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/moved/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/nuked/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/assigned/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/moved/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/dropped/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/deleted/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/unneeded/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/raw": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/normalization": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/normalized": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/loadQueue/size": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/loadQueue/failed": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/loadQueue/counter": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/dropQueue/counter": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/size": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "segment/overShadowed/counter": { + "dimensions": [], + "type": "gauge" + }, + "segment/max": { + "dimensions": [], + "type": "gauge" + }, + "segment/used": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + }, + "segment/usedPercent": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + }, + "jvm/pool/committed": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/init": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/max": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/used": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/counter": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/used": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/capacity": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/mem/init": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/max": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/used": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/committed": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/gc/counter": { + "dimensions": [ + "gcName" + ], + "type": "counter" + }, + "jvm/gc/cpu": { + "dimensions": [ + "gcName" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "ingest/events/buffered": { + "dimensions": [ + "serviceName, bufferCapacity" + ], + "type": "gauge" + }, + "sys/swap/free": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/max": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/pageIn": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/pageOut": { + "dimensions": [], + "type": "gauge" + }, + "sys/disk/write/counter": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/read/counter": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/write/size": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/read/size": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/net/write/size": { + "dimensions": [], + "type": "counter" + }, + "sys/net/read/size": { + "dimensions": [], + "type": "counter" + }, + "sys/fs/used": { + "dimensions": [ + "fsDevName", + "fsDirName", + "fsTypeName", + "fsSysTypeName", + "fsOptions" + ], + "type": "gauge" + }, + "sys/fs/max": { + "dimensions": [ + "fsDevName", + "fsDirName", + "fsTypeName", + "fsSysTypeName", + "fsOptions" + ], + "type": "gauge" + }, + "sys/mem/used": { + "dimensions": [], + "type": "gauge" + }, + "sys/mem/max": { + "dimensions": [], + "type": "gauge" + }, + "sys/storage/used": { + "dimensions": [ + "fsDirName" + ], + "type": "gauge" + }, + "sys/cpu": { + "dimensions": [ + "cpuName", + "cpuTime" + ], + "type": "gauge" + }, + "coordinator-segment/counter": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "historical-segment/counter": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + } +} \ No newline at end of file diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json deleted file mode 100644 index 87cbd8951653..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "ingest/events": [], - "ingest/handoff/failed": [], - "ingest/persists": [], - "ingest/rows/output": [], - "jvm/gc": [], - "jvm/mem": [], - "query/cpu/time": [ - "dataSource", - "type" - ], - "query/node/time": [ - "dataSource", - "type" - ], - "query/node/ttfb": [ - "dataSource", - "type" - ], - "query/partial/time": [ - "dataSource", - "type" - ], - "query/segment/time": [ - "dataSource", - "type" - ], - "query/segmentAndCache/time": [ - "dataSource", - "type" - ], - "query/time": [ - "dataSource", - "type" - ], - "query/wait/time": [ - "dataSource", - "type" - ], - "segment/count": [], - "segment/dropQueue/count": [], - "segment/loadQueue/count": [], - "segment/loadQueue/failed": [], - "segment/loadQueue/size": [], - "segment/scan/pending": [], - "segment/size": [], - "segment/usedPercent": [] -} diff --git a/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java b/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java deleted file mode 100644 index b3cf659176af..000000000000 --- a/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java +++ /dev/null @@ -1,63 +0,0 @@ -package io.druid.emitter.dropwizard; - -import com.fasterxml.jackson.databind.InjectableValues; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.druid.emitter.dropwizard.eventconverters.SendAllDropwizardEventConverter; -import io.druid.emitter.dropwizard.eventconverters.WhiteListBasedConverter; -import io.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; -import io.druid.jackson.DefaultObjectMapper; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.*; - -public class DropwizardEmitterConfigTest { - private ObjectMapper mapper = new DefaultObjectMapper(); - - @Before - public void setUp() - { - mapper.setInjectableValues(new InjectableValues.Std().addValue( - ObjectMapper.class, - new DefaultObjectMapper() - )); - } - - @Test - public void testSerDeserDropwizardEmitterConfig() throws IOException - { - DropwizardEmitterConfig dropwizardEmitterConfig = new DropwizardEmitterConfig(new DropwizardConsoleReporter(),new SendAllDropwizardEventConverter(null,false,false),new HistogramMetricManager(),null); - String dropwizardEmitterConfigString = mapper.writeValueAsString(dropwizardEmitterConfig); - DropwizardEmitterConfig dropwizardEmitterConfigExpected = mapper.reader(DropwizardEmitterConfig.class).readValue( - dropwizardEmitterConfigString - ); - Assert.assertEquals(dropwizardEmitterConfigExpected, dropwizardEmitterConfig); - } - - @Test - public void testSerDeserDruidToDropwizardEventConverter() throws IOException - { - SendAllDropwizardEventConverter sendAllDropwizardEventConverter = new SendAllDropwizardEventConverter("prefix", true, true); - String noopDropwizardEventConverterString = mapper.writeValueAsString(sendAllDropwizardEventConverter); - DruidToDropwizardEventConverter druidToDropwizardEventConverter = mapper.reader(DruidToDropwizardEventConverter.class) - .readValue(noopDropwizardEventConverterString); - Assert.assertEquals(druidToDropwizardEventConverter, sendAllDropwizardEventConverter); - - WhiteListBasedConverter whiteListBasedConverter = new WhiteListBasedConverter( - "prefix", - true, - true, - "", - new DefaultObjectMapper() - ); - String whiteListBasedConverterString = mapper.writeValueAsString(whiteListBasedConverter); - druidToDropwizardEventConverter = mapper.reader(DruidToDropwizardEventConverter.class) - .readValue(whiteListBasedConverterString); - Assert.assertEquals(druidToDropwizardEventConverter, whiteListBasedConverter); - } - - } - - diff --git a/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java b/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java new file mode 100644 index 000000000000..c0f62d75fb23 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import org.apache.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; +import org.apache.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +public class DropwizardEmitterConfigTest +{ + private ObjectMapper mapper = new DefaultObjectMapper(); + + @Before + public void setUp() + { + mapper.setInjectableValues(new InjectableValues.Std().addValue( + ObjectMapper.class, + new DefaultObjectMapper() + )); + } + + @Test + public void testSerDeserDropwizardEmitterConfig() throws IOException + { + DropwizardEmitterConfig dropwizardEmitterConfig = new DropwizardEmitterConfig( + Lists.newArrayList(new DropwizardConsoleReporter(), new DropwizardJMXReporter()), + "my-prefix", + false, + "my/config/path", + false, + null + ); + String dropwizardEmitterConfigString = mapper.writeValueAsString(dropwizardEmitterConfig); + DropwizardEmitterConfig dropwizardEmitterConfigExpected = mapper.readerFor(DropwizardEmitterConfig.class).readValue( + dropwizardEmitterConfigString + ); + Assert.assertEquals(dropwizardEmitterConfigExpected, dropwizardEmitterConfig); + } + +} + + diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index c17439cbf8f2..6717151d0bf3 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -1,24 +1,24 @@ - 4.0.0 diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java index dcbc64baa3ee..9cb773450af2 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java @@ -123,6 +123,9 @@ *

* The RemoteTaskRunner uses ZK for job management and assignment and http for IPC messages. */ + + + //KubernetesTaskRunner public class RemoteTaskRunner implements WorkerTaskRunner, TaskLogStreamer { private static final EmittingLogger log = new EmittingLogger(RemoteTaskRunner.class); diff --git a/pom.xml b/pom.xml index 154847a0d2f4..f2e01944bf38 100644 --- a/pom.xml +++ b/pom.xml @@ -158,6 +158,8 @@ extensions-contrib/influx-extensions extensions-contrib/azure-extensions extensions-contrib/cassandra-storage + extensions-contrib/druid-rocketmq + extensions-contrib/dropwizard-emitter extensions-contrib/cloudfiles-extensions extensions-contrib/graphite-emitter extensions-contrib/distinctcount From 302a7feeda1c07a74bdfb7feb32d6586119737f1 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 29 Mar 2019 01:10:15 +0530 Subject: [PATCH 03/16] Fix teamcity inspections --- .../emitter/dropwizard/DropwizardEmitter.java | 22 +++---------------- .../dropwizard/DropwizardReporter.java | 8 +++---- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java index c32997e3f41d..4fd59dd76fac 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java @@ -32,7 +32,6 @@ import org.apache.druid.java.util.emitter.service.AlertEvent; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import java.io.IOException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -69,12 +68,7 @@ public DropwizardEmitter( public void start() { for (DropwizardReporter reporter : reporters) { - try { - reporter.start(metricsRegistry); - } - catch (IOException e) { - log.error(e, "Error while flushing Dropwizard reporter"); - } + reporter.start(metricsRegistry); } started.set(true); } @@ -156,12 +150,7 @@ private void updateMetric(String name, Number value, DropwizardMetricSpec metric public void flush() { for (DropwizardReporter reporter : reporters) { - try { - reporter.flush(); - } - catch (IOException e) { - log.error(e, "Error while flushing Dropwizard reporter"); - } + reporter.flush(); } } @@ -169,12 +158,7 @@ public void flush() public void close() { for (DropwizardReporter reporter : reporters) { - try { - reporter.close(); - } - catch (IOException e) { - log.error(e, "Error while closing Dropwizard reporter"); - } + reporter.close(); } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java index 4bca4c5fc730..92ffd1f0bc9d 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java @@ -25,8 +25,6 @@ import org.apache.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; import org.apache.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; -import java.io.IOException; - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { @JsonSubTypes.Type(name = "console", value = DropwizardConsoleReporter.class), @@ -35,9 +33,9 @@ public interface DropwizardReporter { - void start(MetricRegistry metricRegistry) throws IOException; + void start(MetricRegistry metricRegistry); - void flush() throws IOException; + void flush(); - void close() throws IOException; + void close(); } From 6f80eece271c5134c61f22e3f3b6404255676ca8 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 26 Jun 2019 23:16:47 +0530 Subject: [PATCH 04/16] review comments --- docs/design/extensions-contrib/dropwizard.md | 7 ++- extensions-contrib/dropwizard-emitter/pom.xml | 32 ++++++------- .../dropwizard/DropwizardConverter.java | 20 +++++--- .../emitter/dropwizard/DropwizardEmitter.java | 48 ++++++++++--------- .../dropwizard/DropwizardEmitterConfig.java | 14 +----- .../reporters/DropwizardConsoleReporter.java | 1 + .../DropwizardEmitterConfigTest.java | 1 - extensions-contrib/graphite-emitter/pom.xml | 32 ++++++------- .../indexing/overlord/RemoteTaskRunner.java | 3 -- 9 files changed, 76 insertions(+), 82 deletions(-) diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index 111f49bd5b0f..0eafc7e54cb1 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -43,7 +43,7 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter |`druid.emitter.dropwizard.prefix`|Optional prefix to be used for metrics name|no|none| |`druid.emitter.dropwizard.includeHost`|Flag to include the hostname as part of the metric name.|no|yes| |`druid.emitter.dropwizard.includeDimensionNames`|Flag to include the dimension names as part of the metric name.|no|yes| -|`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the StatsD type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| +|`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the dropwizard metric type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| |`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| @@ -51,9 +51,8 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter Each metric emitted using Dropwizard must specify a type, one of `[timer, counter, guage, meter, histogram, ]`. Dropwizard Emitter expects this mapping to be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric. -If the user does not specify their own JSON file, a default mapping is used. -All metrics are expected to be mapped. Metrics which are not mapped will log an error. - +If the user does not specify their own JSON file, a [default mapping](https://github.com/apache/incubator-druid/tree/master/extensions-contrib/dropwizard/src/main/resources/defaultMetricDimensions.json) is used. +All metrics are expected to be mapped. Metrics which are not mapped will be ignored. Dropwizard metric path is organized using the following schema: ` : { "dimensions" : , "type" : , "timeUnit" : }` diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml index 03c655a5b5aa..e72c40ea4f09 100644 --- a/extensions-contrib/dropwizard-emitter/pom.xml +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -1,21 +1,21 @@ metricMap; + private final Map metricMap; public DropwizardConverter(ObjectMapper mapper, String dimensionMapPath) { metricMap = readMap(mapper, dimensionMapPath); } + /** + * Filters user dimensions for given metric and adds them to filteredDimensions. + * Returns null if there is no mapping present for the given metric. + */ @Nullable public DropwizardMetricSpec addFilteredUserDims( String service, String metric, Map userDims, - Map builder + Map filteredDimensions ) { /* @@ -58,27 +63,29 @@ public DropwizardMetricSpec addFilteredUserDims( This is because some metrics are reported differently, but with the same name, from different services. */ DropwizardMetricSpec metricSpec = null; - if (metricMap.containsKey(metric)) { - metricSpec = metricMap.get(metric); + DropwizardMetricSpec dropwizardMetricSpec = metricMap.get(metric); + if (dropwizardMetricSpec != null) { + metricSpec = dropwizardMetricSpec; } else if (metricMap.containsKey(service + "-" + metric)) { metricSpec = metricMap.get(service + "-" + metric); } if (metricSpec != null) { for (String dim : metricSpec.getDimensions()) { if (userDims.containsKey(dim)) { - builder.put(dim, userDims.get(dim).toString()); + filteredDimensions.put(dim, userDims.get(dim).toString()); } } return metricSpec; } else { + // No mapping found for given metric, return null return null; } } private Map readMap(ObjectMapper mapper, String dimensionMapPath) { + InputStream is = null; try { - InputStream is; if (Strings.isNullOrEmpty(dimensionMapPath)) { log.info("Using default metric dimension and types"); is = this.getClass().getClassLoader().getResourceAsStream("defaultMetricDimensions.json"); @@ -91,6 +98,7 @@ private Map readMap(ObjectMapper mapper, String di }).readValue(is); } catch (IOException e) { + Closeables.closeQuietly(is); throw new ISE(e, "Failed to parse metric dimensions and types"); } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java index 4fd59dd76fac..92367b8e6db7 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java @@ -36,11 +36,12 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; public class DropwizardEmitter implements Emitter { - private static Logger log = new Logger(DropwizardEmitter.class); + private static final Logger log = new Logger(DropwizardEmitter.class); private final MetricRegistry metricsRegistry = new MetricRegistry(); private final AtomicBoolean started = new AtomicBoolean(false); private final DropwizardConverter converter; @@ -67,15 +68,22 @@ public DropwizardEmitter( @Override public void start() { - for (DropwizardReporter reporter : reporters) { - reporter.start(metricsRegistry); + final boolean alreadyStarted = started.getAndSet(true); + if (!alreadyStarted) { + for (DropwizardReporter reporter : reporters) { + reporter.start(metricsRegistry); + } } - started.set(true); } @Override public void emit(Event event) { + synchronized (started) { + if (!started.get()) { + throw new RejectedExecutionException("Dropwizard emitter Service not started."); + } + } if (event instanceof ServiceMetricEvent) { ServiceMetricEvent metricEvent = (ServiceMetricEvent) event; String host = metricEvent.getHost(); @@ -88,24 +96,15 @@ public void emit(Event event) final DropwizardMetricSpec metricSpec = converter.addFilteredUserDims(service, metric, userDims, dims); if (metricSpec != null) { - if (config.isIncludeDimensionNames()) { - if (config.getPrefix() != null) { - nameBuilder.add(config.getPrefix()); - } - nameBuilder.add("metric=" + metric); - nameBuilder.add("service=" + service); - if (config.getIncludeHost()) { - nameBuilder.add("hostname=" + host); - } - dims.forEach((key, value1) -> nameBuilder.add(key + "=" + value1)); - } else { - nameBuilder.add(metric); - nameBuilder.add(service); - if (config.getIncludeHost()) { - nameBuilder.add(host); - } - nameBuilder.addAll(dims.values()); + if (config.getPrefix() != null) { + nameBuilder.add(config.getPrefix()); } + nameBuilder.add("metric=" + metric); + nameBuilder.add("service=" + service); + if (config.getIncludeHost()) { + nameBuilder.add("hostname=" + host); + } + dims.forEach((key, value1) -> nameBuilder.add(key + "=" + value1)); String fullName = StringUtils.replaceChar(Joiner.on(",").join(nameBuilder.build()), '/', "."); updateMetric(fullName, value, metricSpec); @@ -157,8 +156,11 @@ public void flush() @Override public void close() { - for (DropwizardReporter reporter : reporters) { - reporter.close(); + final boolean wasStarted = started.getAndSet(false); + if (wasStarted) { + for (DropwizardReporter reporter : reporters) { + reporter.close(); + } } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java index 6977e4265dcd..c92ccbe62f31 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java @@ -38,8 +38,6 @@ public class DropwizardEmitterConfig @JsonProperty private final String dimensionMapPath; @JsonProperty - private final Boolean includeDimensionNames; - @JsonProperty private final List alertEmitters; @JsonCreator @@ -48,7 +46,6 @@ public DropwizardEmitterConfig( @JsonProperty("prefix") String prefix, @JsonProperty("includeHost") Boolean includeHost, @JsonProperty("dimensionMapPath") String dimensionMapPath, - @JsonProperty("includeDimensionNames") Boolean includeDimensionNames, @JsonProperty("alertEmitters") List alertEmitters ) { @@ -57,7 +54,6 @@ public DropwizardEmitterConfig( this.alertEmitters = alertEmitters == null ? Collections.emptyList() : alertEmitters; this.includeHost = includeHost != null ? includeHost : true; this.dimensionMapPath = dimensionMapPath; - this.includeDimensionNames = includeDimensionNames != null ? includeDimensionNames : true; } @JsonProperty @@ -84,12 +80,6 @@ public String getDimensionMapPath() return dimensionMapPath; } - @JsonProperty - public Boolean isIncludeDimensionNames() - { - return includeDimensionNames; - } - @JsonProperty public List getAlertEmitters() { @@ -110,14 +100,13 @@ public boolean equals(Object o) Objects.equals(prefix, that.prefix) && Objects.equals(includeHost, that.includeHost) && Objects.equals(dimensionMapPath, that.dimensionMapPath) && - Objects.equals(includeDimensionNames, that.includeDimensionNames) && Objects.equals(alertEmitters, that.alertEmitters); } @Override public int hashCode() { - return Objects.hash(reporters, prefix, includeHost, dimensionMapPath, includeDimensionNames, alertEmitters); + return Objects.hash(reporters, prefix, includeHost, dimensionMapPath, alertEmitters); } @Override @@ -128,7 +117,6 @@ public String toString() ", prefix='" + prefix + '\'' + ", includeHost=" + includeHost + ", dimensionMapPath='" + dimensionMapPath + '\'' + - ", includeDimensionNames=" + includeDimensionNames + ", alertEmitters=" + alertEmitters + '}'; } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java index 8b8559bfb0e1..5b85c87c21f4 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java @@ -85,6 +85,7 @@ public void start(MetricRegistry metricRegistry) @Override public void flush() { + } @Override diff --git a/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java b/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java index c0f62d75fb23..2b3b667a829a 100644 --- a/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java +++ b/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java @@ -52,7 +52,6 @@ public void testSerDeserDropwizardEmitterConfig() throws IOException "my-prefix", false, "my/config/path", - false, null ); String dropwizardEmitterConfigString = mapper.writeValueAsString(dropwizardEmitterConfig); diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 6717151d0bf3..36a3f8c58148 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -1,21 +1,21 @@ * The RemoteTaskRunner uses ZK for job management and assignment and http for IPC messages. */ - - - //KubernetesTaskRunner public class RemoteTaskRunner implements WorkerTaskRunner, TaskLogStreamer { private static final EmittingLogger log = new EmittingLogger(RemoteTaskRunner.class); From c6cdf70128c756d246c66cd711367f81a28454f4 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 26 Jun 2019 23:36:10 +0530 Subject: [PATCH 05/16] more review comments --- docs/design/extensions-contrib/dropwizard.md | 1 - .../dropwizard/DropwizardEmitterModule.java | 15 ++++++++++----- .../reporters/DropwizardConsoleReporter.java | 4 ++-- .../reporters/DropwizardJMXReporter.java | 8 +++++--- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index 0eafc7e54cb1..b66f557f5b56 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -42,7 +42,6 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter |`druid.emitter.dropwizard.reporters`|List of dropwizard reporters to be used.|yes|none| |`druid.emitter.dropwizard.prefix`|Optional prefix to be used for metrics name|no|none| |`druid.emitter.dropwizard.includeHost`|Flag to include the hostname as part of the metric name.|no|yes| -|`druid.emitter.dropwizard.includeDimensionNames`|Flag to include the dimension names as part of the metric name.|no|yes| |`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the dropwizard metric type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| |`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java index 0ad7000f655c..395bc9011ce7 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Key; @@ -35,6 +34,7 @@ import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; public class DropwizardEmitterModule implements DruidModule { @@ -61,10 +61,15 @@ public Emitter getEmitter( final Injector injector ) { - List alertEmitters = Lists.transform( - dropwizardEmitterConfig.getAlertEmitters(), - s -> injector.getInstance(Key.get(Emitter.class, Names.named(s))) - ); + List alertEmitters = dropwizardEmitterConfig.getAlertEmitters() + .stream() + .map(s -> injector.getInstance( + Key.get( + Emitter.class, + Names.named(s) + ))) + .collect(Collectors.toList()); + return new DropwizardEmitter(dropwizardEmitterConfig, mapper, alertEmitters); } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java index 5b85c87c21f4..79cbf47fe033 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java @@ -85,13 +85,13 @@ public void start(MetricRegistry metricRegistry) @Override public void flush() { - + // no-op } @Override public void close() { - + consoleReporter.stop(); } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java index dcbb8027a94e..e3cd3e5d2d76 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java @@ -31,6 +31,7 @@ public class DropwizardJMXReporter implements DropwizardReporter { private String domain = "org.apache.druid"; + private JmxReporter reporter; @JsonProperty public String getDomain() @@ -41,20 +42,21 @@ public String getDomain() @Override public void start(MetricRegistry metricRegistry) { - final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry) - .inDomain(domain).build(); + reporter = JmxReporter.forRegistry(metricRegistry) + .inDomain(domain).build(); reporter.start(); } @Override public void flush() { + // no-op } @Override public void close() { - + reporter.close(); } @Override From f5c81fc8591a30ea605f8110dcbb9c2555a7c595 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 27 Jun 2019 01:03:35 +0530 Subject: [PATCH 06/16] add limit to max number of gauges --- docs/design/extensions-contrib/dropwizard.md | 1 + .../emitter/dropwizard/DropwizardEmitter.java | 5 ++- .../dropwizard/DropwizardEmitterConfig.java | 19 +++++++-- .../druid/emitter/dropwizard/GaugesCache.java | 39 +++++++++++++++++++ .../DropwizardEmitterConfigTest.java | 3 +- 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/GaugesCache.java diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index b66f557f5b56..a5615e7e811d 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -44,6 +44,7 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter |`druid.emitter.dropwizard.includeHost`|Flag to include the hostname as part of the metric name.|no|yes| |`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the dropwizard metric type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| |`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| +|`druid.emitter.dropwizard.maxGaugeCount`| Maximum size of distinct gauge metrics to be cached at any time. |no| 100K| ### Druid to Dropwizard Event Conversion diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java index 92367b8e6db7..b78dab807a26 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java @@ -32,10 +32,10 @@ import org.apache.druid.java.util.emitter.service.AlertEvent; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; @@ -50,7 +50,7 @@ public class DropwizardEmitter implements Emitter private final DropwizardEmitterConfig config; // Note: the gauges do not represent the actual instantaneous value for the metrics. // Instead they have the last known value for it. - private final ConcurrentHashMap gagues = new ConcurrentHashMap<>(); + private final Map gagues; public DropwizardEmitter( DropwizardEmitterConfig config, @@ -62,6 +62,7 @@ public DropwizardEmitter( this.config = config; this.reporters = config.getReporters(); this.converter = new DropwizardConverter(mapper, config.getDimensionMapPath()); + this.gagues = Collections.synchronizedMap(new GaugesCache<>(config.getMaxGaugeCount())); } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java index c92ccbe62f31..8dabf8cf9c28 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java @@ -29,6 +29,7 @@ public class DropwizardEmitterConfig { + private static int DEFAULT_MAX_GAUGE_COUNT = 100_000; @JsonProperty private final List reporters; @JsonProperty @@ -39,6 +40,8 @@ public class DropwizardEmitterConfig private final String dimensionMapPath; @JsonProperty private final List alertEmitters; + @JsonProperty + private final int maxGaugeCount; @JsonCreator public DropwizardEmitterConfig( @@ -46,7 +49,8 @@ public DropwizardEmitterConfig( @JsonProperty("prefix") String prefix, @JsonProperty("includeHost") Boolean includeHost, @JsonProperty("dimensionMapPath") String dimensionMapPath, - @JsonProperty("alertEmitters") List alertEmitters + @JsonProperty("alertEmitters") List alertEmitters, + @JsonProperty("maxGaugeCount") Integer maxGaugeCount ) { this.reporters = reporters; @@ -54,6 +58,7 @@ public DropwizardEmitterConfig( this.alertEmitters = alertEmitters == null ? Collections.emptyList() : alertEmitters; this.includeHost = includeHost != null ? includeHost : true; this.dimensionMapPath = dimensionMapPath; + this.maxGaugeCount = maxGaugeCount == null ? DEFAULT_MAX_GAUGE_COUNT : maxGaugeCount; } @JsonProperty @@ -86,6 +91,12 @@ public List getAlertEmitters() return alertEmitters; } + @JsonProperty + public int getMaxGaugeCount() + { + return maxGaugeCount; + } + @Override public boolean equals(Object o) { @@ -96,7 +107,8 @@ public boolean equals(Object o) return false; } DropwizardEmitterConfig that = (DropwizardEmitterConfig) o; - return Objects.equals(reporters, that.reporters) && + return maxGaugeCount == that.maxGaugeCount && + Objects.equals(reporters, that.reporters) && Objects.equals(prefix, that.prefix) && Objects.equals(includeHost, that.includeHost) && Objects.equals(dimensionMapPath, that.dimensionMapPath) && @@ -106,7 +118,7 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(reporters, prefix, includeHost, dimensionMapPath, alertEmitters); + return Objects.hash(reporters, prefix, includeHost, dimensionMapPath, alertEmitters, maxGaugeCount); } @Override @@ -118,6 +130,7 @@ public String toString() ", includeHost=" + includeHost + ", dimensionMapPath='" + dimensionMapPath + '\'' + ", alertEmitters=" + alertEmitters + + ", maxGaugeCount=" + maxGaugeCount + '}'; } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/GaugesCache.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/GaugesCache.java new file mode 100644 index 000000000000..0f465f8f7103 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/GaugesCache.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.emitter.dropwizard; + +import java.util.LinkedHashMap; +import java.util.Map; + +class GaugesCache extends LinkedHashMap +{ + private int capacity; + + public GaugesCache(int capacity) + { + this.capacity = capacity; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) + { + return this.size() > this.capacity; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java b/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java index 2b3b667a829a..ed9b9f809317 100644 --- a/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java +++ b/extensions-contrib/dropwizard-emitter/src/test/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java @@ -52,7 +52,8 @@ public void testSerDeserDropwizardEmitterConfig() throws IOException "my-prefix", false, "my/config/path", - null + null, + 400 ); String dropwizardEmitterConfigString = mapper.writeValueAsString(dropwizardEmitterConfig); DropwizardEmitterConfig dropwizardEmitterConfigExpected = mapper.readerFor(DropwizardEmitterConfig.class).readValue( From 2ec065a6628fb631caf59e214f67b89eff4818e7 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 27 Jun 2019 01:29:37 +0530 Subject: [PATCH 07/16] update pom version --- extensions-contrib/dropwizard-emitter/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml index e72c40ea4f09..189a7498e5de 100644 --- a/extensions-contrib/dropwizard-emitter/pom.xml +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -25,7 +25,7 @@ org.apache.druid druid - 0.15.0-incubating-SNAPSHOT + 0.16.0-incubating-SNAPSHOT ../../pom.xml From 0bfeb0a5864022d432deb3709325927c10a16802 Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 8 Jul 2019 18:25:34 +0530 Subject: [PATCH 08/16] fix pom --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index f2e01944bf38..1a6f6a70a7de 100644 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,6 @@ extensions-contrib/influx-extensions extensions-contrib/azure-extensions extensions-contrib/cassandra-storage - extensions-contrib/druid-rocketmq extensions-contrib/dropwizard-emitter extensions-contrib/cloudfiles-extensions extensions-contrib/graphite-emitter From 93037bfbefeaa2cf457ff47c057ed4289aef5354 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Mon, 2 Sep 2019 12:43:26 +0530 Subject: [PATCH 09/16] review comments --- docs/design/extensions-contrib/dropwizard.md | 20 +++++++++---------- extensions-contrib/dropwizard-emitter/pom.xml | 9 +++++++-- .../dropwizard/DropwizardConverter.java | 7 +++---- .../emitter/dropwizard/DropwizardEmitter.java | 3 +++ .../dropwizard/DropwizardEmitterModule.java | 2 -- .../reporters/DropwizardConsoleReporter.java | 2 -- .../reporters/DropwizardJMXReporter.java | 4 +--- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index a5615e7e811d..c7ba0ed26c88 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -36,12 +36,12 @@ These metrics can be emitted using either Console or JMX reporter. ## Configuration All the configuration parameters for Dropwizard emitter are under `druid.emitter.dropwizard`. - + |property|description|required?|default| |--------|-----------|---------|-------| -|`druid.emitter.dropwizard.reporters`|List of dropwizard reporters to be used.|yes|none| +|`druid.emitter.dropwizard.reporters`|List of dropwizard reporters to be used. Here is a list of [Supported Reporters](#supported-dropwizard-reporters)|yes|none| |`druid.emitter.dropwizard.prefix`|Optional prefix to be used for metrics name|no|none| -|`druid.emitter.dropwizard.includeHost`|Flag to include the hostname as part of the metric name.|no|yes| +|`druid.emitter.dropwizard.includeHost`|Flag to include the host and port as part of the metric name.|no|yes| |`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the dropwizard metric type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| |`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| |`druid.emitter.dropwizard.maxGaugeCount`| Maximum size of distinct gauge metrics to be cached at any time. |no| 100K| @@ -49,7 +49,7 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter ### Druid to Dropwizard Event Conversion -Each metric emitted using Dropwizard must specify a type, one of `[timer, counter, guage, meter, histogram, ]`. Dropwizard Emitter expects this mapping to +Each metric emitted using Dropwizard must specify a type, one of `[timer, counter, guage, meter, histogram]`. Dropwizard Emitter expects this mapping to be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric. If the user does not specify their own JSON file, a [default mapping](https://github.com/apache/incubator-druid/tree/master/extensions-contrib/dropwizard/src/main/resources/defaultMetricDimensions.json) is used. All metrics are expected to be mapped. Metrics which are not mapped will be ignored. @@ -58,10 +58,10 @@ Dropwizard metric path is organized using the following schema: ` : { "dimensions" : , "type" : , "timeUnit" : }` e.g. - -`"query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer", "timeUnit": "MILLISECONDS"}` -`"segment/scan/pending" : { "dimensions" : [], "type" : "gauge"}` - +```json +"query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer", "timeUnit": "MILLISECONDS"}, +"segment/scan/pending" : { "dimensions" : [], "type" : "gauge"} +``` For most use-cases, the default mapping is sufficient. @@ -69,7 +69,7 @@ For most use-cases, the default mapping is sufficient. #### JMX Reporter Used to report druid metrics via JMX. -```json +``` druid.emitter.dropwizard.reporter={"type":"jmx"} @@ -78,7 +78,7 @@ druid.emitter.dropwizard.reporter={"type":"jmx"} #### Console Reporter Used to print Druid Metrics to console logs. -```json +``` druid.emitter.dropwizard.reporter={"type":"console","emitIntervalInSecs":30}"} diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml index 189a7498e5de..3adf9ec0ac51 100644 --- a/extensions-contrib/dropwizard-emitter/pom.xml +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -25,7 +25,7 @@ org.apache.druid druid - 0.16.0-incubating-SNAPSHOT + 0.17.0-incubating-SNAPSHOT ../../pom.xml @@ -50,7 +50,12 @@ io.dropwizard.metrics metrics-core - 3.1.0 + ${dropwizard.metrics.version} + + + io.dropwizard.metrics + metrics-jmx + ${dropwizard.metrics.version} junit diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java index 27226eb53f33..9bf343fe3122 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java @@ -58,10 +58,9 @@ public DropwizardMetricSpec addFilteredUserDims( Map filteredDimensions ) { - /* - Find the metric in the map. If we cant find it try to look it up prefixed by the service name. - This is because some metrics are reported differently, but with the same name, from different services. - */ + + // Find the metric in the map. If we cant find it try to look it up prefixed by the service name. + // This is because some metrics are reported differently, but with the same name, from different services. DropwizardMetricSpec metricSpec = null; DropwizardMetricSpec dropwizardMetricSpec = metricMap.get(metric); if (dropwizardMetricSpec != null) { diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java index b78dab807a26..0816cef39bdb 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java @@ -143,6 +143,9 @@ private void updateMetric(String name, Number value, DropwizardMetricSpec metric if (prevVal == null) { metricsRegistry.register(name, (Gauge) () -> gagues.get(name)); } + break; + default: + throw new ISE("Unknown Metric Type [%s]", metricSpec.getType()); } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java index 395bc9011ce7..a700b88050cb 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java @@ -28,7 +28,6 @@ import com.google.inject.name.Named; import com.google.inject.name.Names; import org.apache.druid.guice.JsonConfigProvider; -import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.initialization.DruidModule; import org.apache.druid.java.util.emitter.core.Emitter; @@ -53,7 +52,6 @@ public void configure(Binder binder) } @Provides - @ManageLifecycle @Named(EMITTER_TYPE) public Emitter getEmitter( DropwizardEmitterConfig dropwizardEmitterConfig, diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java index 79cbf47fe033..336be6d9684b 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java @@ -22,12 +22,10 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.druid.emitter.dropwizard.DropwizardReporter; import java.util.concurrent.TimeUnit; -@JsonTypeName("console") public class DropwizardConsoleReporter implements DropwizardReporter { private long emitIntervalInSecs; diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java index e3cd3e5d2d76..258490caf5ad 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java @@ -19,15 +19,13 @@ package org.apache.druid.emitter.dropwizard.reporters; -import com.codahale.metrics.JmxReporter; import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.jmx.JmxReporter; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.druid.emitter.dropwizard.DropwizardReporter; import java.util.Objects; -@JsonTypeName("jmx") public class DropwizardJMXReporter implements DropwizardReporter { private String domain = "org.apache.druid"; From 9cb2c89274832866b76175071b1104ccdba5f742 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Mon, 2 Sep 2019 12:50:45 +0530 Subject: [PATCH 10/16] review comment --- docs/design/extensions-contrib/dropwizard.md | 546 ++++++++++++++++++- 1 file changed, 545 insertions(+), 1 deletion(-) diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index c7ba0ed26c88..67d622dfcdce 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -51,7 +51,7 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter Each metric emitted using Dropwizard must specify a type, one of `[timer, counter, guage, meter, histogram]`. Dropwizard Emitter expects this mapping to be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric. -If the user does not specify their own JSON file, a [default mapping](https://github.com/apache/incubator-druid/tree/master/extensions-contrib/dropwizard/src/main/resources/defaultMetricDimensions.json) is used. +If the user does not specify their own JSON file, a [default mapping](#default-metrics-mapping) is used. All metrics are expected to be mapped. Metrics which are not mapped will be ignored. Dropwizard metric path is organized using the following schema: @@ -82,4 +82,548 @@ Used to print Druid Metrics to console logs. druid.emitter.dropwizard.reporter={"type":"console","emitIntervalInSecs":30}"} +``` + +### Default Metrics Mapping +```json +{ + "query/time": { + "dimensions": [ + "dataSource", + "type" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/time": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/ttfb": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/node/backpressure": { + "dimensions": [ + "server" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/intervalChunk/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/segment/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/wait/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "segment/scan/pending": { + "dimensions": [], + "type": "gauge" + }, + "query/segmentAndCache/time": { + "dimensions": [], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "query/cpu/time": { + "dimensions": [ + "dataSource", + "type" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "query/cache/delta/numEntries": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/sizeBytes": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/hits": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/misses": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/evictions": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/hitRate": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/averageBytes": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/timeouts": { + "dimensions": [], + "type": "counter" + }, + "query/cache/delta/errors": { + "dimensions": [], + "type": "counter" + }, + "query/cache/total/numEntries": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/sizeBytes": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/hits": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/misses": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/evictions": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/hitRate": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/averageBytes": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/timeouts": { + "dimensions": [], + "type": "gauge" + }, + "query/cache/total/errors": { + "dimensions": [], + "type": "gauge" + }, + "ingest/events/thrownAway": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/unparseable": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/duplicate": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/events/processed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/rows/output": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/persist/counter": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/persist/time": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "ingest/persist/cpu": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "ingest/persist/backPressure": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "ingest/persist/failed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/handoff/failed": { + "dimensions": [ + "dataSource" + ], + "type": "counter" + }, + "ingest/merge/time": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "ingest/merge/cpu": { + "dimensions": [ + "dataSource" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "task/run/time": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "timer", + "timeUnit": "MILLISECONDS" + }, + "segment/added/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/moved/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/nuked/bytes": { + "dimensions": [ + "dataSource", + "taskType" + ], + "type": "counter" + }, + "segment/assigned/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/moved/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/dropped/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/deleted/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/unneeded/counter": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/raw": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/normalization": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/cost/normalized": { + "dimensions": [ + "tier" + ], + "type": "counter" + }, + "segment/loadQueue/size": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/loadQueue/failed": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/loadQueue/counter": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/dropQueue/counter": { + "dimensions": [ + "server" + ], + "type": "gauge" + }, + "segment/size": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "segment/overShadowed/counter": { + "dimensions": [], + "type": "gauge" + }, + "segment/max": { + "dimensions": [], + "type": "gauge" + }, + "segment/used": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + }, + "segment/usedPercent": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + }, + "jvm/pool/committed": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/init": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/max": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/pool/used": { + "dimensions": [ + "poolKind", + "poolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/counter": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/used": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/bufferpool/capacity": { + "dimensions": [ + "bufferPoolName" + ], + "type": "gauge" + }, + "jvm/mem/init": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/max": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/used": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/mem/committed": { + "dimensions": [ + "memKind" + ], + "type": "gauge" + }, + "jvm/gc/counter": { + "dimensions": [ + "gcName" + ], + "type": "counter" + }, + "jvm/gc/cpu": { + "dimensions": [ + "gcName" + ], + "type": "timer", + "timeUnit": "NANOSECONDS" + }, + "ingest/events/buffered": { + "dimensions": [ + "serviceName, bufferCapacity" + ], + "type": "gauge" + }, + "sys/swap/free": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/max": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/pageIn": { + "dimensions": [], + "type": "gauge" + }, + "sys/swap/pageOut": { + "dimensions": [], + "type": "gauge" + }, + "sys/disk/write/counter": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/read/counter": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/write/size": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/disk/read/size": { + "dimensions": [ + "fsDevName" + ], + "type": "counter" + }, + "sys/net/write/size": { + "dimensions": [], + "type": "counter" + }, + "sys/net/read/size": { + "dimensions": [], + "type": "counter" + }, + "sys/fs/used": { + "dimensions": [ + "fsDevName", + "fsDirName", + "fsTypeName", + "fsSysTypeName", + "fsOptions" + ], + "type": "gauge" + }, + "sys/fs/max": { + "dimensions": [ + "fsDevName", + "fsDirName", + "fsTypeName", + "fsSysTypeName", + "fsOptions" + ], + "type": "gauge" + }, + "sys/mem/used": { + "dimensions": [], + "type": "gauge" + }, + "sys/mem/max": { + "dimensions": [], + "type": "gauge" + }, + "sys/storage/used": { + "dimensions": [ + "fsDirName" + ], + "type": "gauge" + }, + "sys/cpu": { + "dimensions": [ + "cpuName", + "cpuTime" + ], + "type": "gauge" + }, + "coordinator-segment/counter": { + "dimensions": [ + "dataSource" + ], + "type": "gauge" + }, + "historical-segment/counter": { + "dimensions": [ + "dataSource", + "tier", + "priority" + ], + "type": "gauge" + } +} ``` \ No newline at end of file From 2f7a8cdcec509f4d5cb2a7135a9d845a63d4f20f Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Mon, 2 Sep 2019 12:56:00 +0530 Subject: [PATCH 11/16] review comments --- .../reporters/DropwizardConsoleReporter.java | 10 ++++++++++ .../dropwizard/reporters/DropwizardJMXReporter.java | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java index 336be6d9684b..9b0a072faf0d 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java @@ -130,4 +130,14 @@ public int hashCode() result = 31 * result + (consoleReporter != null ? consoleReporter.hashCode() : 0); return result; } + + @Override + public String toString() + { + return "DropwizardConsoleReporter{" + + "emitIntervalInSecs=" + emitIntervalInSecs + + ", rates=" + rates + + ", durations=" + durations + + '}'; + } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java index 258490caf5ad..654ff67a5d1a 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java @@ -75,4 +75,12 @@ public int hashCode() { return Objects.hash(domain); } + + @Override + public String toString() + { + return "DropwizardJMXReporter{" + + "domain='" + domain + '\'' + + '}'; + } } From 8775f16928de6d28ea5256900acba23a5f290915 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Tue, 3 Sep 2019 01:09:41 +0530 Subject: [PATCH 12/16] fix broken doc link review comments review comments --- .idea/misc.xml | 2 +- docs/design/extensions-contrib/dropwizard.md | 9 +-- .../emitter/dropwizard/DropwizardEmitter.java | 59 ++++++++++++++----- .../dropwizard/DropwizardEmitterConfig.java | 21 ++++--- 4 files changed, 62 insertions(+), 29 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index 61e7e5b21e77..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -87,4 +87,4 @@ - \ No newline at end of file + diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index 67d622dfcdce..dd5ca292c93c 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -24,7 +24,7 @@ title: "Dropwizard metrics emitter" # Dropwizard Emitter -To use this extension, make sure to [include](../../operations/including-extensions.html) `dropwizard-emitter` extension. +To use this extension, make sure to [include](../../development/extensions.md#loading-extensions) `dropwizard-emitter` extension. ## Introduction @@ -44,7 +44,7 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter |`druid.emitter.dropwizard.includeHost`|Flag to include the host and port as part of the metric name.|no|yes| |`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the dropwizard metric type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| |`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| -|`druid.emitter.dropwizard.maxGaugeCount`| Maximum size of distinct gauge metrics to be cached at any time. |no| 100K| +|`druid.emitter.dropwizard.maxMetricsRegistrySize`| Maximum size of metrics registry to be cached at any time. |no| 100Mb| ### Druid to Dropwizard Event Conversion @@ -71,7 +71,7 @@ For most use-cases, the default mapping is sufficient. Used to report druid metrics via JMX. ``` -druid.emitter.dropwizard.reporter={"type":"jmx"} +druid.emitter.dropwizard.reporters={"type":"jmx"} ``` @@ -80,11 +80,12 @@ Used to print Druid Metrics to console logs. ``` -druid.emitter.dropwizard.reporter={"type":"console","emitIntervalInSecs":30}"} +druid.emitter.dropwizard.reporters={"type":"console","emitIntervalInSecs":30}"} ``` ### Default Metrics Mapping +Latest default metrics mapping can be foung [here] (https://github.com/apache/incubator-druid/tree/master/extensions-contrib/dropwizard/src/main/resources/defaultMetricDimensions.json) ```json { "query/time": { diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java index 0816cef39bdb..8b6b48144b00 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java @@ -20,8 +20,11 @@ package org.apache.druid.emitter.dropwizard; import com.codahale.metrics.Gauge; +import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import org.apache.druid.java.util.common.ISE; @@ -32,25 +35,22 @@ import org.apache.druid.java.util.emitter.service.AlertEvent; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; public class DropwizardEmitter implements Emitter { private static final Logger log = new Logger(DropwizardEmitter.class); - private final MetricRegistry metricsRegistry = new MetricRegistry(); + private final MetricRegistry metricsRegistry; private final AtomicBoolean started = new AtomicBoolean(false); private final DropwizardConverter converter; private final List alertEmitters; private final List reporters; private final DropwizardEmitterConfig config; - // Note: the gauges do not represent the actual instantaneous value for the metrics. - // Instead they have the last known value for it. - private final Map gagues; public DropwizardEmitter( DropwizardEmitterConfig config, @@ -62,7 +62,18 @@ public DropwizardEmitter( this.config = config; this.reporters = config.getReporters(); this.converter = new DropwizardConverter(mapper, config.getDimensionMapPath()); - this.gagues = Collections.synchronizedMap(new GaugesCache<>(config.getMaxGaugeCount())); + final Cache metricsRegistryCache = Caffeine.newBuilder() + .recordStats() + .maximumSize(config.getMaxMetricsRegistrySize()) + .build(); + metricsRegistry = new MetricRegistry() + { + @Override + protected ConcurrentMap buildMap() + { + return metricsRegistryCache.asMap(); + } + }; } @@ -100,12 +111,12 @@ public void emit(Event event) if (config.getPrefix() != null) { nameBuilder.add(config.getPrefix()); } - nameBuilder.add("metric=" + metric); - nameBuilder.add("service=" + service); + nameBuilder.add(StringUtils.format("metric=%s", metric)); + nameBuilder.add(StringUtils.format("service=%s", service)); if (config.getIncludeHost()) { - nameBuilder.add("hostname=" + host); + nameBuilder.add(StringUtils.format("hostname=%s", host)); } - dims.forEach((key, value1) -> nameBuilder.add(key + "=" + value1)); + dims.forEach((key, value1) -> nameBuilder.add(StringUtils.format("%s=%s", key, value1))); String fullName = StringUtils.replaceChar(Joiner.on(",").join(nameBuilder.build()), '/', "."); updateMetric(fullName, value, metricSpec); @@ -138,11 +149,8 @@ private void updateMetric(String name, Number value, DropwizardMetricSpec metric metricsRegistry.histogram(name).update(value.longValue()); break; case gauge: - Number prevVal = gagues.put(name, value); - // It is the first time we get this metric, register it with metricsRegistry - if (prevVal == null) { - metricsRegistry.register(name, (Gauge) () -> gagues.get(name)); - } + SettableGauge gauge = (SettableGauge) metricsRegistry.gauge(name, () -> new SettableGauge(value)); + gauge.setValue(value); break; default: throw new ISE("Unknown Metric Type [%s]", metricSpec.getType()); @@ -168,4 +176,25 @@ public void close() } } + private static class SettableGauge implements Gauge + { + private Number value; + + public SettableGauge(Number value) + { + this.value = value; + } + + public void setValue(Number value) + { + this.value = value; + } + + @Override + public Number getValue() + { + return null; + } + } + } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java index 8dabf8cf9c28..c724df10f7fa 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterConfig.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; import java.util.Collections; import java.util.List; @@ -29,7 +30,8 @@ public class DropwizardEmitterConfig { - private static int DEFAULT_MAX_GAUGE_COUNT = 100_000; + // default to 100 Mb + private static int DEFAULT_METRICS_REGISTRY_SIZE = 100_000_000; @JsonProperty private final List reporters; @JsonProperty @@ -41,7 +43,7 @@ public class DropwizardEmitterConfig @JsonProperty private final List alertEmitters; @JsonProperty - private final int maxGaugeCount; + private final int maxMetricsRegistrySize; @JsonCreator public DropwizardEmitterConfig( @@ -50,15 +52,16 @@ public DropwizardEmitterConfig( @JsonProperty("includeHost") Boolean includeHost, @JsonProperty("dimensionMapPath") String dimensionMapPath, @JsonProperty("alertEmitters") List alertEmitters, - @JsonProperty("maxGaugeCount") Integer maxGaugeCount + @JsonProperty("metricsRegistrySize") Integer maxMetricsRegistrySize ) { + Preconditions.checkArgument(reporters != null && !reporters.isEmpty()); this.reporters = reporters; this.prefix = prefix; this.alertEmitters = alertEmitters == null ? Collections.emptyList() : alertEmitters; this.includeHost = includeHost != null ? includeHost : true; this.dimensionMapPath = dimensionMapPath; - this.maxGaugeCount = maxGaugeCount == null ? DEFAULT_MAX_GAUGE_COUNT : maxGaugeCount; + this.maxMetricsRegistrySize = maxMetricsRegistrySize == null ? DEFAULT_METRICS_REGISTRY_SIZE : maxMetricsRegistrySize; } @JsonProperty @@ -92,9 +95,9 @@ public List getAlertEmitters() } @JsonProperty - public int getMaxGaugeCount() + public int getMaxMetricsRegistrySize() { - return maxGaugeCount; + return maxMetricsRegistrySize; } @Override @@ -107,7 +110,7 @@ public boolean equals(Object o) return false; } DropwizardEmitterConfig that = (DropwizardEmitterConfig) o; - return maxGaugeCount == that.maxGaugeCount && + return maxMetricsRegistrySize == that.maxMetricsRegistrySize && Objects.equals(reporters, that.reporters) && Objects.equals(prefix, that.prefix) && Objects.equals(includeHost, that.includeHost) && @@ -118,7 +121,7 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(reporters, prefix, includeHost, dimensionMapPath, alertEmitters, maxGaugeCount); + return Objects.hash(reporters, prefix, includeHost, dimensionMapPath, alertEmitters, maxMetricsRegistrySize); } @Override @@ -130,7 +133,7 @@ public String toString() ", includeHost=" + includeHost + ", dimensionMapPath='" + dimensionMapPath + '\'' + ", alertEmitters=" + alertEmitters + - ", maxGaugeCount=" + maxGaugeCount + + ", maxMetricsRegistrySize=" + maxMetricsRegistrySize + '}'; } } From 7a660784ddd60954349c5d8ccc489f301d0c8661 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Wed, 25 Sep 2019 14:17:07 +0530 Subject: [PATCH 13/16] review comments --- .../druid/emitter/dropwizard/DropwizardConverter.java | 3 ++- .../apache/druid/emitter/dropwizard/DropwizardEmitter.java | 2 +- .../apache/druid/emitter/dropwizard/DropwizardReporter.java | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java index 9bf343fe3122..216c47153842 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java @@ -97,8 +97,9 @@ private Map readMap(ObjectMapper mapper, String di }).readValue(is); } catch (IOException e) { - Closeables.closeQuietly(is); throw new ISE(e, "Failed to parse metric dimensions and types"); + } finally { + Closeables.closeQuietly(is); } } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java index 8b6b48144b00..5baa1b5da245 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitter.java @@ -193,7 +193,7 @@ public void setValue(Number value) @Override public Number getValue() { - return null; + return value; } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java index 92ffd1f0bc9d..f0cc0811ae22 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java @@ -25,6 +25,8 @@ import org.apache.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; import org.apache.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; +import java.io.Flushable; + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { @JsonSubTypes.Type(name = "console", value = DropwizardConsoleReporter.class), @@ -35,6 +37,10 @@ public interface DropwizardReporter void start(MetricRegistry metricRegistry); + /** + * Used for reporters that choose to buffer events to trigger flushing of buffered events. + * It should be a non-blocking operation. + */ void flush(); void close(); From 170bf838302cd143e382c58dc4d8d5db044c251b Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Thu, 26 Sep 2019 13:44:35 +0530 Subject: [PATCH 14/16] fix checkstyle --- .../apache/druid/emitter/dropwizard/DropwizardConverter.java | 3 ++- .../apache/druid/emitter/dropwizard/DropwizardReporter.java | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java index 216c47153842..7c1194726b31 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardConverter.java @@ -98,7 +98,8 @@ private Map readMap(ObjectMapper mapper, String di } catch (IOException e) { throw new ISE(e, "Failed to parse metric dimensions and types"); - } finally { + } + finally { Closeables.closeQuietly(is); } } diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java index f0cc0811ae22..9bb32481460e 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardReporter.java @@ -25,8 +25,6 @@ import org.apache.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; import org.apache.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; -import java.io.Flushable; - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { @JsonSubTypes.Type(name = "console", value = DropwizardConsoleReporter.class), From 7211d7045dc9bc60fdf5b8cee8dc4783de003d02 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Thu, 26 Sep 2019 16:06:31 +0530 Subject: [PATCH 15/16] more spell check fixes --- docs/design/extensions-contrib/dropwizard.md | 4 +-- extensions-contrib/dropwizard-emitter/pom.xml | 35 +++++++++++++++++++ website/.spelling | 3 ++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index dd5ca292c93c..283954b304ef 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -44,7 +44,7 @@ All the configuration parameters for Dropwizard emitter are under `druid.emitter |`druid.emitter.dropwizard.includeHost`|Flag to include the host and port as part of the metric name.|no|yes| |`druid.emitter.dropwizard.dimensionMapPath`|Path to JSON file defining the dropwizard metric type, and desired dimensions for every Druid metric|no|Default mapping provided. See below.| |`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| -|`druid.emitter.dropwizard.maxMetricsRegistrySize`| Maximum size of metrics registry to be cached at any time. |no| 100Mb| +|`druid.emitter.dropwizard.maxMetricsRegistrySize`| Maximum size of metrics registry to be cached at any time. |no| 100 Mb| ### Druid to Dropwizard Event Conversion @@ -85,7 +85,7 @@ druid.emitter.dropwizard.reporters={"type":"console","emitIntervalInSecs":30}"} ``` ### Default Metrics Mapping -Latest default metrics mapping can be foung [here] (https://github.com/apache/incubator-druid/tree/master/extensions-contrib/dropwizard/src/main/resources/defaultMetricDimensions.json) +Latest default metrics mapping can be found [here] (https://github.com/apache/incubator-druid/tree/master/extensions-contrib/dropwizard/src/main/resources/defaultMetricDimensions.json) ```json { "query/time": { diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml index 3adf9ec0ac51..bca1ad7ccf0c 100644 --- a/extensions-contrib/dropwizard-emitter/pom.xml +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -80,5 +80,40 @@ test-jar test + + com.fasterxml.jackson.core + jackson-annotations + provided + + + org.apache.curator + curator-client + provided + + + com.google.inject + guice + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.fasterxml.jackson.core + jackson-core + provided + + + com.github.ben-manes.caffeine + caffeine + provided + + + com.google.guava + guava + provided + diff --git a/website/.spelling b/website/.spelling index 0b875ed486ea..14a4c833276c 100644 --- a/website/.spelling +++ b/website/.spelling @@ -54,6 +54,7 @@ Double.NEGATIVE_INFINITY Double.NEGATIVE_INFINITY. Double.POSITIVE_INFINITY Double.POSITIVE_INFINITY. +Dropwizard DruidSQL EC2 EMR @@ -89,6 +90,7 @@ JDK JDK7 JDK8 JKS +JMX JRE JS JSON @@ -759,6 +761,7 @@ statsd-emitter coords dimName maxCoords +Mb minCoords - ../docs/development/javascript.md Metaspace From 22ef36da16559a258b428a2e05acf8f03ad691ff Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Tue, 1 Oct 2019 23:20:54 +0530 Subject: [PATCH 16/16] fix travis failures --- extensions-contrib/dropwizard-emitter/pom.xml | 8 ++++++-- website/.spelling | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml index bca1ad7ccf0c..def2c20c0dab 100644 --- a/extensions-contrib/dropwizard-emitter/pom.xml +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -77,8 +77,7 @@ org.apache.druid druid-processing ${project.parent.version} - test-jar - test + provided com.fasterxml.jackson.core @@ -115,5 +114,10 @@ guava provided + + com.google.code.findbugs + jsr305 + provided + diff --git a/website/.spelling b/website/.spelling index 14a4c833276c..1e06e42e2366 100644 --- a/website/.spelling +++ b/website/.spelling @@ -55,6 +55,7 @@ Double.NEGATIVE_INFINITY. Double.POSITIVE_INFINITY Double.POSITIVE_INFINITY. Dropwizard +dropwizard DruidSQL EC2 EMR