Fix issue with poorly registered metrics upon node restarts

thelastpickle · Feb 17, 2018 · 00c2cc7 · 00c2cc7
1 parent 07a897c
commit 00c2cc7
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 8 deletions.
diff --git a/src/server/src/main/java/io/cassandrareaper/jmx/JmxConnectionFactory.java b/src/server/src/main/java/io/cassandrareaper/jmx/JmxConnectionFactory.java
@@ -61,9 +61,17 @@ public JmxConnectionFactory(MetricRegistry metricRegistry) {
   }
 
   private void registerConnectionsGauge() {
-    this.metricRegistry.register(
-        MetricRegistry.name(JmxConnectionFactory.class, "openJmxConnections"),
-        (Gauge<Integer>) () -> JMX_CONNECTIONS.size());
+    try {
+      if (!this.metricRegistry
+          .getGauges()
+          .containsKey(MetricRegistry.name(JmxConnectionFactory.class, "openJmxConnections"))) {
+        this.metricRegistry.register(
+            MetricRegistry.name(JmxConnectionFactory.class, "openJmxConnections"),
+            (Gauge<Integer>) () -> JMX_CONNECTIONS.size());
+      }
+    } catch (IllegalArgumentException e) {
+      LOG.warn("Cannot create openJmxConnections metric gauge", e);
+    }
   }
 
   protected JmxProxy connect(Optional<RepairStatusHandler> handler, String host, int connectionTimeout)
@@ -85,10 +93,10 @@ protected JmxProxy connect(Optional<RepairStatusHandler> handler, String host, i
       if (!proxy.isConnectionAlive()) {
         LOG.info("Adding new JMX Proxy for host {}", host);
         JMX_CONNECTIONS.put(host, provider.apply(host)).close();
-        proxy.close();
       }
       return JMX_CONNECTIONS.get(host);
     } catch (RuntimeException ex) {
+      LOG.error("Failed creating a new JMX connection to {}", host, ex);
       // unpack any exception behind JmxConnectionProvider.apply(..)
       if (ex.getCause() instanceof InterruptedException) {
         throw (InterruptedException) ex.getCause();

diff --git a/src/server/src/main/java/io/cassandrareaper/jmx/JmxProxyImpl.java b/src/server/src/main/java/io/cassandrareaper/jmx/JmxProxyImpl.java
@@ -1042,9 +1042,19 @@ public String getColumnFamily() {
   }
 
   private void registerConnectionsGauge() {
-    this.metricRegistry.register(
-        MetricRegistry.name(
-            JmxProxyImpl.class, this.clusterName, this.host, "repairStatusHandlers"),
-        (Gauge<Integer>) () -> this.repairStatusHandlers.size());
+    try {
+      if (!this.metricRegistry
+          .getGauges()
+          .containsKey(
+              MetricRegistry.name(
+                  JmxProxyImpl.class, this.clusterName, this.host, "repairStatusHandlers"))) {
+        this.metricRegistry.register(
+            MetricRegistry.name(
+                JmxProxyImpl.class, this.clusterName, this.host, "repairStatusHandlers"),
+            (Gauge<Integer>) () -> this.repairStatusHandlers.size());
+      }
+    } catch (IllegalArgumentException e) {
+      LOG.warn("Cannot create connection gauge for node {}", this.host, e);
+    }
   }
 }