Merge pull request #627 from tbak/master

1.x Additional metrics for client side heartbeat/registry staleness monitoring

eureka-client/src/main/java/com/netflix/discovery/DiscoveryClient.java

@@ -67,6 +67,7 @@
 import com.netflix.discovery.shared.Applications;
 import com.netflix.discovery.shared.EurekaJerseyClient;
 import com.netflix.discovery.shared.EurekaJerseyClient.EurekaJerseyClientBuilder;
+import com.netflix.discovery.util.ThresholdLevelsMetric;
 import com.netflix.eventbus.spi.EventBus;
 import com.netflix.servo.annotations.DataSourceType;
 import com.netflix.servo.monitor.Counter;
@@ -80,6 +81,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static com.netflix.discovery.EurekaClientNames.METRIC_REGISTRY_PREFIX;
+
 /**
  * The class that is instrumental for interactions with <tt>Eureka Server</tt>.
  *
@@ -172,6 +175,8 @@ public class DiscoveryClient implements EurekaClient {
     private ApplicationInfoManager.StatusChangeListener statusChangeListener;
     private volatile long lastSuccessfulRegistryFetchTimestamp = -1;
     private volatile long lastSuccessfulHeartbeatTimestamp = -1;
+    private final ThresholdLevelsMetric heartbeatStalenessMonitor;
+    private final ThresholdLevelsMetric registryStalenessMonitor;
 
     private enum Action {
         Register, Cancel, Renew, Refresh, Refresh_Delta
@@ -397,6 +402,8 @@ public synchronized BackupRegistry get() {
         } catch (Throwable e) {
             logger.warn("Cannot register timers", e);
         }
+        this.heartbeatStalenessMonitor = new ThresholdLevelsMetric(this, "lastHeartbeatSec_", new long[]{15L, 30L, 60L, 120L, 240L, 480L});
+        this.registryStalenessMonitor = new ThresholdLevelsMetric(this, "lastUpdateSec_", new long[]{15L, 30L, 60L, 120L, 240L, 480L});
 
         // This is a bit of hack to allow for existing code using DiscoveryManager.getInstance()
         // to work with DI'd DiscoveryClient
@@ -838,6 +845,8 @@ public void shutdown() {
         if (discoveryJerseyClient != null) {
             discoveryJerseyClient.destroyResources();
         }
+        heartbeatStalenessMonitor.shutdown();
+        registryStalenessMonitor.shutdown();
     }
 
     /**
@@ -2112,15 +2121,19 @@ protected void fireEvent(DiscoveryEvent event) {
         }
     }
 
-    @com.netflix.servo.annotations.Monitor(name = "lastSuccessfulHeartbeatTimePeriod",
+    @com.netflix.servo.annotations.Monitor(name = METRIC_REGISTRY_PREFIX + "lastSuccessfulHeartbeatTimePeriod",
             description = "How much time has passed from last successful heartbeat", type = DataSourceType.GAUGE)
     public long getLastSuccessfulHeartbeatTimePeriod() {
-        return lastSuccessfulHeartbeatTimestamp < 0 ? 0 : System.currentTimeMillis() - lastSuccessfulHeartbeatTimestamp;
+        long delay = lastSuccessfulHeartbeatTimestamp < 0 ? 0 : System.currentTimeMillis() - lastSuccessfulHeartbeatTimestamp;
+        heartbeatStalenessMonitor.update(delay);
+        return delay;
     }
 
-    @com.netflix.servo.annotations.Monitor(name = "lastSuccessfulRegistryFetchTimePeriod",
+    @com.netflix.servo.annotations.Monitor(name = METRIC_REGISTRY_PREFIX + "lastSuccessfulRegistryFetchTimePeriod",
             description = "How much time has passed from last successful local registry update", type = DataSourceType.GAUGE)
     public long getLastSuccessfulRegistryFetchTimePeriod() {
-        return lastSuccessfulRegistryFetchTimestamp < 0 ? 0 : System.currentTimeMillis() - lastSuccessfulRegistryFetchTimestamp;
+        long delay = lastSuccessfulRegistryFetchTimestamp < 0 ? 0 : System.currentTimeMillis() - lastSuccessfulRegistryFetchTimestamp;
+        registryStalenessMonitor.update(delay);
+        return delay;
     }
 }

eureka-client/src/main/java/com/netflix/discovery/EurekaClientNames.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2015 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.discovery;
+
+/**
+ * @author Tomasz Bak
+ */
+public final class EurekaClientNames {
+
+    /**
+     * Eureka metric names consist of three parts [source].[component].[detailed name]:
+     * <ul>
+     *     <li>source - fixed to eurekaClient (and eurekaServer on the server side)</li>
+     *     <li>component - Eureka component, like registry cache</li>
+     *     <li>detailed name - a detailed metric name explaining its purpose</li>
+     * </ul>
+     */
+    public static final String METRIC_PREFIX = "eurekaClient.";
+
+    public static final String METRIC_REGISTRY_PREFIX = METRIC_PREFIX + "registry.";
+
+    private EurekaClientNames() {
+    }
+}

eureka-client/src/main/java/com/netflix/discovery/util/ThresholdLevelsMetric.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2015 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.discovery.util;
+
+import com.netflix.discovery.EurekaClientNames;
+import com.netflix.servo.DefaultMonitorRegistry;
+import com.netflix.servo.monitor.LongGauge;
+import com.netflix.servo.monitor.MonitorConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A collection of gauges that represent different threshold levels over which measurement is mapped to.
+ * Value 1 denotes a lowest threshold level that is reached.
+ * For example eureka client registry data staleness defines thresholds 30s, 60s, 120s, 240s, 480s. Delay of 90s
+ * would be mapped to gauge values {30s=0, 60s=1, 120=0, 240s=0, 480s=0, unlimited=0}.
+ *
+ * @author Tomasz Bak
+ */
+public class ThresholdLevelsMetric {
+
+    private static final Logger logger = LoggerFactory.getLogger(ThresholdLevelsMetric.class);
+
+    private final long[] levels;
+    private final LongGauge[] gauges;
+
+    public ThresholdLevelsMetric(Object owner, String prefix, long[] levels) {
+        this.levels = levels;
+        this.gauges = new LongGauge[levels.length];
+        for (int i = 0; i < levels.length; i++) {
+            String name = EurekaClientNames.METRIC_REGISTRY_PREFIX + prefix + String.format("%05d", levels[i]);
+            MonitorConfig config = new MonitorConfig.Builder(name)
+                    .withTag("class", owner.getClass().getName())
+                    .build();
+            gauges[i] = new LongGauge(config);
+
+            try {
+                DefaultMonitorRegistry.getInstance().register(gauges[i]);
+            } catch (Throwable e) {
+                logger.warn("Cannot register metric " + name, e);
+            }
+        }
+    }
+
+    public void update(long delayMs) {
+        long delaySec = delayMs / 1000;
+        long matchedIdx;
+        if (levels[0] > delaySec) {
+            matchedIdx = -1;
+        } else {
+            matchedIdx = levels.length - 1;
+            for (int i = 0; i < levels.length - 1; i++) {
+                if (levels[i] <= delaySec && delaySec < levels[i + 1]) {
+                    matchedIdx = i;
+                    break;
+                }
+            }
+        }
+        for (int i = 0; i < levels.length; i++) {
+            if (i == matchedIdx) {
+                gauges[i].set(1L);
+            } else {
+                gauges[i].set(0L);
+            }
+        }
+    }
+
+    public void shutdown() {
+        for (LongGauge gauge : gauges) {
+            try {
+                DefaultMonitorRegistry.getInstance().unregister(gauge);
+            } catch (Throwable ignore) {
+            }
+        }
+    }
+}

eureka-core/src/main/java/com/netflix/eureka/GzipEncodingEnforcingFilter.java

@@ -34,7 +34,7 @@ public void init(FilterConfig filterConfig) throws ServletException {
     @Override
     public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
         HttpServletRequest httpRequest = (HttpServletRequest) request;
-        if (httpRequest.getMethod() == "GET") {
+        if ("GET".equals(httpRequest.getMethod())) {
             String acceptEncoding = httpRequest.getHeader(HttpHeaders.ACCEPT_ENCODING);
             if (acceptEncoding == null) {
                 chain.doFilter(addGzipAcceptEncoding(httpRequest), response);