From b5a799470a1b58780fb5393bb237f46024bb0ef6 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 9 Jan 2019 11:52:16 +0100 Subject: [PATCH 01/40] P2P roundtriptime metric init --- build.gradle | 2 + .../src/main/java/bisq/monitor/Metric.java | 2 +- .../bisq/monitor/metric/P2PRoundTripTime.java | 170 ++++++++++++++++++ .../bisq/monitor/P2PRoundTripTimeTests.java | 123 +++++++++++++ 4 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java create mode 100644 monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java diff --git a/build.gradle b/build.gradle index 9768c4e40ca..216ee7127d7 100644 --- a/build.gradle +++ b/build.gradle @@ -322,6 +322,8 @@ configure(project(':monitor')) { } dependencies { + compile project(':p2p') + compile project(':core') compile "org.slf4j:slf4j-api:$slf4jVersion" compile "ch.qos.logback:logback-core:$logbackVersion" compile "ch.qos.logback:logback-classic:$logbackVersion" diff --git a/monitor/src/main/java/bisq/monitor/Metric.java b/monitor/src/main/java/bisq/monitor/Metric.java index ed537e7784f..7838a3b7a38 100644 --- a/monitor/src/main/java/bisq/monitor/Metric.java +++ b/monitor/src/main/java/bisq/monitor/Metric.java @@ -51,7 +51,7 @@ private void disable() { /** * enable execution */ - private void enable() { + protected void enable() { shutdown = false; thread = new Thread(this); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java new file mode 100644 index 00000000000..bbf38a05bcb --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -0,0 +1,170 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor.metric; + +import java.io.File; +import java.util.Properties; +import java.util.Random; + +import org.jetbrains.annotations.NotNull; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; + +import bisq.common.proto.network.NetworkEnvelope; +import bisq.core.proto.network.CoreNetworkProtoResolver; +import bisq.monitor.Metric; +import bisq.monitor.Reporter; +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.network.Connection; +import bisq.network.p2p.network.MessageListener; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.network.NewTor; +import bisq.network.p2p.network.SetupListener; +import bisq.network.p2p.network.TorNetworkNode; +import bisq.network.p2p.peers.keepalive.messages.Ping; +import bisq.network.p2p.peers.keepalive.messages.Pong; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class P2PRoundTripTime extends Metric implements MessageListener, SetupListener { + + private NetworkNode networkNode; + private final File torWorkingDirectory = new File("metric_p2pRoundTripTime"); + private int nonce; + private long start; + private Boolean ready = false; + + public P2PRoundTripTime(Reporter reporter) { + super(reporter); + } + + @Override + protected void enable() { + Thread sepp = new Thread(new Runnable() { + + @Override + public void run() { + synchronized (ready) { + while (!ready) + try { + ready.wait(); + } catch (InterruptedException ignore) { + } + P2PRoundTripTime.super.enable(); + } + } + }); + sepp.start(); + } + + @Override + public void configure(Properties properties) { + super.configure(properties); + + networkNode = new TorNetworkNode(9052, new CoreNetworkProtoResolver(), false, + new NewTor(torWorkingDirectory, "", "", null)); + networkNode.start(this); + } + + /** + * synchronization helper. Required because directly closing the + * HiddenServiceSocket in its ReadyListener causes a deadlock + */ + private void await() { + synchronized (torWorkingDirectory) { + try { + torWorkingDirectory.wait(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + private void proceed() { + synchronized (torWorkingDirectory) { + torWorkingDirectory.notify(); + } + } + + @Override + protected void execute() { + nonce = new Random().nextInt(); + start = System.currentTimeMillis(); + SettableFuture future = networkNode.sendMessage(new NodeAddress("s67qglwhkgkyvr74.onion:8000"), + new Ping(nonce, 5)); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Connection connection) { + log.debug("Send ping to " + connection + " succeeded."); + connection.addMessageListener(P2PRoundTripTime.this); + } + + @Override + public void onFailure(@NotNull Throwable throwable) { + log.error("Sending ping failed. That is expected if the peer is offline.\n\tException=" + + throwable.getMessage()); + } + }); + + await(); + } + + @Override + public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { + if (networkEnvelope instanceof Pong) { + Pong pong = (Pong) networkEnvelope; + if (pong.getRequestNonce() == nonce) { + reporter.report(System.currentTimeMillis() - start, "bisq." + getName()); + } else { + log.warn("Nonce not matching. That should never happen.\n\t" + + "We drop that message. nonce={} / requestNonce={}", + nonce, pong.getRequestNonce()); + } + connection.removeMessageListener(this); + proceed(); + } + } + + @Override + public void onTorNodeReady() { + // TODO Auto-generated method stub + } + + @Override + public void onHiddenServicePublished() { + synchronized (ready) { + ready.notify(); + ready = true; + } + } + + @Override + public void onSetupFailed(Throwable throwable) { + // TODO Auto-generated method stub + + } + + @Override + public void onRequestCustomBridges() { + // TODO Auto-generated method stub + + } +} diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java new file mode 100644 index 00000000000..7a8d7ea3d6c --- /dev/null +++ b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java @@ -0,0 +1,123 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import bisq.monitor.metric.P2PRoundTripTime; +import bisq.monitor.reporter.ConsoleReporter; + +import org.berndpruenster.netlayer.tor.NativeTor; +import org.berndpruenster.netlayer.tor.Tor; +import org.berndpruenster.netlayer.tor.TorCtlException; + +import java.io.File; + +import java.util.Map; +import java.util.Properties; + +import org.junit.Assert; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Test the round trip time metric against the hidden service of tor project.org. + * + * @author Florian Reimair + */ +public class P2PRoundTripTimeTests { + + /** + * A dummy Reporter for development purposes. + */ + private class DummyReporter extends ConsoleReporter { + + private Map results; + + @Override + public void report(long value) { + Assert.fail(); + } + + public Map hasResults() { + return results; + } + + @Override + public void report(Map values) { + super.report(values); + results = values; + } + + @Override + public void report(Map values, String prefix) { + report(values); + } + + @Override + public void report(long value, String prefix) { + report(value); + } + } + + @ParameterizedTest + @ValueSource(strings = {"default", "3", "4", "10"}) + public void run(String sampleSize) throws Exception { + DummyReporter reporter = new DummyReporter(); + + // configure + Properties configuration = new Properties(); + configuration.put("P2PRoundTripTime.enabled", "true"); + configuration.put("P2PRoundTripTime.run.interval", "2"); + if (!"default".equals(sampleSize)) + configuration.put("TorRoundTripTime.run.sampleSize", sampleSize); + // torproject.org hidden service + configuration.put("P2PRoundTripTime.run.hosts", "http://expyuzz4wqqyqhjn.onion:80"); + + Metric DUT = new P2PRoundTripTime(reporter); + // start + DUT.configure(configuration); + + // give it some time to start and then stop + Thread.sleep(100000); + + DUT.shutdown(); + DUT.join(); + + // observe results + Map results = reporter.hasResults(); + Assert.assertFalse(results.isEmpty()); + Assert.assertEquals(results.get("sampleSize"), sampleSize.equals("default") ? "1" : sampleSize); + + Integer p25 = Integer.valueOf(results.get("p25")); + Integer p50 = Integer.valueOf(results.get("p50")); + Integer p75 = Integer.valueOf(results.get("p75")); + Integer min = Integer.valueOf(results.get("min")); + Integer max = Integer.valueOf(results.get("max")); + Integer average = Integer.valueOf(results.get("average")); + + Assert.assertTrue(0 < min); + Assert.assertTrue(min <= p25 && p25 <= p50); + Assert.assertTrue(p50 <= p75); + Assert.assertTrue(p75 <= max); + Assert.assertTrue(min <= average && average <= max); + } +} From 985c353a4dfd17f66f35394d16d9258bdc4fa06c Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 21 Jan 2019 16:33:01 +0100 Subject: [PATCH 02/40] Add missing Version information --- .../java/bisq/monitor/metric/P2PRoundTripTime.java | 2 ++ .../java/bisq/monitor/P2PRoundTripTimeTests.java | 13 +++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index bbf38a05bcb..06637e6f4ca 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -26,6 +26,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; +import bisq.common.app.Version; import bisq.common.proto.network.NetworkEnvelope; import bisq.core.proto.network.CoreNetworkProtoResolver; import bisq.monitor.Metric; @@ -79,6 +80,7 @@ public void configure(Properties properties) { networkNode = new TorNetworkNode(9052, new CoreNetworkProtoResolver(), false, new NewTor(torWorkingDirectory, "", "", null)); + Version.setBaseCryptoNetworkId(1); // set to BTC_MAINNET networkNode.start(this); } diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java index 7a8d7ea3d6c..30896e7eab4 100644 --- a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java @@ -25,7 +25,7 @@ import org.berndpruenster.netlayer.tor.TorCtlException; import java.io.File; - +import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -67,14 +67,11 @@ public void report(Map values) { results = values; } - @Override - public void report(Map values, String prefix) { - report(values); - } - @Override public void report(long value, String prefix) { - report(value); + HashMap result = new HashMap<>(); + result.put(prefix, String.valueOf(value)); + report(result); } } @@ -97,7 +94,7 @@ public void run(String sampleSize) throws Exception { DUT.configure(configuration); // give it some time to start and then stop - Thread.sleep(100000); + Thread.sleep(5000); DUT.shutdown(); DUT.join(); From 2d9ec11e36fb4a4b7eb60ea1ce564187de187d32 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 21 Jan 2019 16:57:46 +0100 Subject: [PATCH 03/40] P2PRttMetric learns about sample size --- .../java/bisq/monitor/StatisticsHelper.java | 66 +++++++++++++++++++ .../bisq/monitor/metric/P2PRoundTripTime.java | 50 +++++++++----- .../bisq/monitor/metric/TorRoundTripTime.java | 31 +-------- .../bisq/monitor/P2PRoundTripTimeTests.java | 15 +++-- 4 files changed, 109 insertions(+), 53 deletions(-) create mode 100644 monitor/src/main/java/bisq/monitor/StatisticsHelper.java diff --git a/monitor/src/main/java/bisq/monitor/StatisticsHelper.java b/monitor/src/main/java/bisq/monitor/StatisticsHelper.java new file mode 100644 index 00000000000..e16c0c84d6b --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/StatisticsHelper.java @@ -0,0 +1,66 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.LongSummaryStatistics; +import java.util.Map; + +/** + * Calculates average, max, min, p25, p50, p75 off of a list of samples and + * throws in the sample size for good measure. + * + * @author Florian Reimair + */ +public class StatisticsHelper { + + public static Map process(List samples) { + + // aftermath + Collections.sort(samples); + + // - average, max, min , sample size + LongSummaryStatistics statistics = samples.stream().mapToLong(val -> val).summaryStatistics(); + + Map results = new HashMap<>(); + results.put("average", String.valueOf(Math.round(statistics.getAverage()))); + results.put("max", String.valueOf(statistics.getMax())); + results.put("min", String.valueOf(statistics.getMin())); + results.put("sampleSize", String.valueOf(statistics.getCount())); + + // - p25, median, p75 + Integer[] percentiles = new Integer[] { 25, 50, 75 }; + for (Integer percentile : percentiles) { + double rank = statistics.getCount() * percentile / 100; + Long percentileValue; + if (samples.size() <= rank + 1) + percentileValue = samples.get(samples.size() - 1); + else if (Math.floor(rank) == rank) + percentileValue = samples.get((int) rank); + else + percentileValue = Math.round(samples.get((int) Math.floor(rank)) + + (samples.get((int) (Math.floor(rank) + 1)) - samples.get((int) Math.floor(rank))) + / (rank - Math.floor(rank))); + results.put("p" + percentile, String.valueOf(percentileValue)); + } + + return results; + } +} diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 06637e6f4ca..253e24f3343 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -18,6 +18,8 @@ package bisq.monitor.metric; import java.io.File; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; import java.util.Random; @@ -31,6 +33,7 @@ import bisq.core.proto.network.CoreNetworkProtoResolver; import bisq.monitor.Metric; import bisq.monitor.Reporter; +import bisq.monitor.StatisticsHelper; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; @@ -45,11 +48,13 @@ @Slf4j public class P2PRoundTripTime extends Metric implements MessageListener, SetupListener { + private static final String SAMPLE_SIZE = "run.sampleSize"; private NetworkNode networkNode; private final File torWorkingDirectory = new File("metric_p2pRoundTripTime"); private int nonce; private long start; private Boolean ready = false; + private List samples; public P2PRoundTripTime(Reporter reporter) { super(reporter); @@ -107,26 +112,35 @@ private void proceed() { @Override protected void execute() { - nonce = new Random().nextInt(); - start = System.currentTimeMillis(); - SettableFuture future = networkNode.sendMessage(new NodeAddress("s67qglwhkgkyvr74.onion:8000"), - new Ping(nonce, 5)); - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(Connection connection) { - log.debug("Send ping to " + connection + " succeeded."); - connection.addMessageListener(P2PRoundTripTime.this); - } + // init sample bucket + samples = new ArrayList<>(); - @Override - public void onFailure(@NotNull Throwable throwable) { - log.error("Sending ping failed. That is expected if the peer is offline.\n\tException=" - + throwable.getMessage()); - } - }); + while (samples.size() < Integer.parseInt(configuration.getProperty(SAMPLE_SIZE, "1"))) { + nonce = new Random().nextInt(); + start = System.currentTimeMillis(); + SettableFuture future = networkNode.sendMessage(new NodeAddress("s67qglwhkgkyvr74.onion:8000"), + new Ping(nonce, 5)); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Connection connection) { + connection.addMessageListener(P2PRoundTripTime.this); + log.debug("Send ping to " + connection + " succeeded."); + } + + @Override + public void onFailure(@NotNull Throwable throwable) { + log.error("Sending ping failed. That is expected if the peer is offline.\n\tException=" + + throwable.getMessage()); + } + }); + + await(); + } - await(); + // report + reporter.report(StatisticsHelper.process(samples), "bisq." + getName()); } @Override @@ -134,7 +148,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof Pong) { Pong pong = (Pong) networkEnvelope; if (pong.getRequestNonce() == nonce) { - reporter.report(System.currentTimeMillis() - start, "bisq." + getName()); + samples.add(System.currentTimeMillis() - start); } else { log.warn("Nonce not matching. That should never happen.\n\t" + "We drop that message. nonce={} / requestNonce={}", diff --git a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java index ab5f6da5466..a0b94735a91 100644 --- a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java @@ -19,6 +19,7 @@ import bisq.monitor.Metric; import bisq.monitor.Reporter; +import bisq.monitor.StatisticsHelper; import org.berndpruenster.netlayer.tor.Tor; import org.berndpruenster.netlayer.tor.TorCtlException; @@ -84,36 +85,8 @@ protected void execute() { socket.close(); } - // aftermath - Collections.sort(samples); - - // - average, max, min , sample size - LongSummaryStatistics statistics = samples.stream().mapToLong(val -> val).summaryStatistics(); - - Map results = new HashMap<>(); - results.put("average", String.valueOf(Math.round(statistics.getAverage()))); - results.put("max", String.valueOf(statistics.getMax())); - results.put("min", String.valueOf(statistics.getMin())); - results.put("sampleSize", String.valueOf(statistics.getCount())); - - // - p25, median, p75 - Integer[] percentiles = new Integer[]{25, 50, 75}; - for (Integer percentile : percentiles) { - double rank = statistics.getCount() * percentile / 100; - Long percentileValue; - if (samples.size() <= rank + 1) - percentileValue = samples.get(samples.size() - 1); - else if (Math.floor(rank) == rank) - percentileValue = samples.get((int) rank); - else - percentileValue = Math.round(samples.get((int) Math.floor(rank)) - + (samples.get((int) (Math.floor(rank) + 1)) - samples.get((int) Math.floor(rank))) - / (rank - Math.floor(rank))); - results.put("p" + percentile, String.valueOf(percentileValue)); - } - // report - reporter.report(results, "bisq." + getName()); + reporter.report(StatisticsHelper.process(samples), "bisq." + getName()); } } catch (TorCtlException | IOException e) { // TODO Auto-generated catch block diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java index 30896e7eab4..5e3846b5e8d 100644 --- a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java @@ -63,15 +63,18 @@ public Map hasResults() { @Override public void report(Map values) { - super.report(values); - results = values; + Assert.fail(); } @Override public void report(long value, String prefix) { - HashMap result = new HashMap<>(); - result.put(prefix, String.valueOf(value)); - report(result); + Assert.fail(); + } + + @Override + public void report(Map values, String prefix) { + super.report(values, prefix); + results = values; } } @@ -85,7 +88,7 @@ public void run(String sampleSize) throws Exception { configuration.put("P2PRoundTripTime.enabled", "true"); configuration.put("P2PRoundTripTime.run.interval", "2"); if (!"default".equals(sampleSize)) - configuration.put("TorRoundTripTime.run.sampleSize", sampleSize); + configuration.put("P2PRoundTripTime.run.sampleSize", sampleSize); // torproject.org hidden service configuration.put("P2PRoundTripTime.run.hosts", "http://expyuzz4wqqyqhjn.onion:80"); From 515247e2d29b8fc505d06f295ac19fcf3e6a2d68 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 21 Jan 2019 17:08:33 +0100 Subject: [PATCH 04/40] P2PRttMetric knows multiple targets --- .../bisq/monitor/metric/P2PRoundTripTime.java | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 253e24f3343..bf99012bed5 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -18,6 +18,7 @@ package bisq.monitor.metric; import java.io.File; +import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.Properties; @@ -49,6 +50,7 @@ public class P2PRoundTripTime extends Metric implements MessageListener, SetupListener { private static final String SAMPLE_SIZE = "run.sampleSize"; + private static final String HOSTS = "run.hosts"; private NetworkNode networkNode; private final File torWorkingDirectory = new File("metric_p2pRoundTripTime"); private int nonce; @@ -113,34 +115,44 @@ private void proceed() { @Override protected void execute() { - // init sample bucket - samples = new ArrayList<>(); - - while (samples.size() < Integer.parseInt(configuration.getProperty(SAMPLE_SIZE, "1"))) { - nonce = new Random().nextInt(); - start = System.currentTimeMillis(); - SettableFuture future = networkNode.sendMessage(new NodeAddress("s67qglwhkgkyvr74.onion:8000"), - new Ping(nonce, 5)); + // for each configured host + for (String current : configuration.getProperty(HOSTS, "").split(",")) { + try { + // parse Url + URL tmp = new URL(current); + NodeAddress target = new NodeAddress(tmp.getHost(), tmp.getPort()); + + // init sample bucket + samples = new ArrayList<>(); + + while (samples.size() < Integer.parseInt(configuration.getProperty(SAMPLE_SIZE, "1"))) { + nonce = new Random().nextInt(); + start = System.currentTimeMillis(); + SettableFuture future = networkNode.sendMessage(target, new Ping(nonce, 42)); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Connection connection) { + connection.addMessageListener(P2PRoundTripTime.this); + log.debug("Send ping to " + connection + " succeeded."); + } - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(Connection connection) { - connection.addMessageListener(P2PRoundTripTime.this); - log.debug("Send ping to " + connection + " succeeded."); - } + @Override + public void onFailure(@NotNull Throwable throwable) { + log.error("Sending ping failed. That is expected if the peer is offline.\n\tException=" + + throwable.getMessage()); + } + }); - @Override - public void onFailure(@NotNull Throwable throwable) { - log.error("Sending ping failed. That is expected if the peer is offline.\n\tException=" - + throwable.getMessage()); + await(); } - }); - await(); + // report + reporter.report(StatisticsHelper.process(samples), "bisq." + getName() + "." + target); + } catch (Exception e) { + e.printStackTrace(); + } } - - // report - reporter.report(StatisticsHelper.process(samples), "bisq." + getName()); } @Override From ad2de68140ff8d4a27d6d5d7c4afc4e77e45c314 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 21 Jan 2019 17:17:29 +0100 Subject: [PATCH 05/40] Cleanup --- monitor/src/main/java/bisq/monitor/Monitor.java | 1 + .../java/bisq/monitor/metric/P2PRoundTripTime.java | 6 +++--- .../java/bisq/monitor/P2PRoundTripTimeTests.java | 13 ++----------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java index f69bfd9739e..1b85e1553d8 100644 --- a/monitor/src/main/java/bisq/monitor/Monitor.java +++ b/monitor/src/main/java/bisq/monitor/Monitor.java @@ -66,6 +66,7 @@ public static void main(String[] args) throws Throwable { * @throws Exception */ private void start() throws Throwable { + // start Tor Tor.setDefault(new NativeTor(new File("monitor/monitor-tor"), null, null, false)); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index bf99012bed5..d7f7ff005f3 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -60,6 +60,8 @@ public class P2PRoundTripTime extends Metric implements MessageListener, SetupLi public P2PRoundTripTime(Reporter reporter) { super(reporter); + + Version.setBaseCryptoNetworkId(1); // set to BTC_MAINNET } @Override @@ -87,13 +89,11 @@ public void configure(Properties properties) { networkNode = new TorNetworkNode(9052, new CoreNetworkProtoResolver(), false, new NewTor(torWorkingDirectory, "", "", null)); - Version.setBaseCryptoNetworkId(1); // set to BTC_MAINNET networkNode.start(this); } /** - * synchronization helper. Required because directly closing the - * HiddenServiceSocket in its ReadyListener causes a deadlock + * synchronization helper. */ private void await() { synchronized (torWorkingDirectory) { diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java index 5e3846b5e8d..9db652bcd89 100644 --- a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java @@ -20,24 +20,14 @@ import bisq.monitor.metric.P2PRoundTripTime; import bisq.monitor.reporter.ConsoleReporter; -import org.berndpruenster.netlayer.tor.NativeTor; -import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; - -import java.io.File; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.junit.Assert; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import static com.google.common.base.Preconditions.checkNotNull; - /** * Test the round trip time metric against the hidden service of tor project.org. * @@ -97,7 +87,8 @@ public void run(String sampleSize) throws Exception { DUT.configure(configuration); // give it some time to start and then stop - Thread.sleep(5000); + while (!DUT.enabled()) + Thread.sleep(2000); DUT.shutdown(); DUT.join(); From 3c6566b65f836ffe4890d66bf1f5a0693dab0cca Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 21 Jan 2019 18:21:04 +0100 Subject: [PATCH 06/40] P2PRtt Metric has configurable Tor proxy port --- .../src/main/java/bisq/monitor/metric/P2PRoundTripTime.java | 4 ++-- monitor/src/main/resources/metrics.properties | 6 ++++++ .../src/test/java/bisq/monitor/P2PRoundTripTimeTests.java | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index d7f7ff005f3..566d70302c0 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -51,6 +51,7 @@ public class P2PRoundTripTime extends Metric implements MessageListener, SetupLi private static final String SAMPLE_SIZE = "run.sampleSize"; private static final String HOSTS = "run.hosts"; + private static final String TOR_PROXY_PORT = "run.torProxyPort"; private NetworkNode networkNode; private final File torWorkingDirectory = new File("metric_p2pRoundTripTime"); private int nonce; @@ -86,8 +87,7 @@ public void run() { @Override public void configure(Properties properties) { super.configure(properties); - - networkNode = new TorNetworkNode(9052, new CoreNetworkProtoResolver(), false, + networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9052")), new CoreNetworkProtoResolver(), false, new NewTor(torWorkingDirectory, "", "", null)); networkNode.start(this); } diff --git a/monitor/src/main/resources/metrics.properties b/monitor/src/main/resources/metrics.properties index 482796cd94d..5e5a714014b 100644 --- a/monitor/src/main/resources/metrics.properties +++ b/monitor/src/main/resources/metrics.properties @@ -24,6 +24,12 @@ TorHiddenServiceStartupTime.run.interval=100 TorHiddenServiceStartupTime.run.localPort=90501 TorHiddenServiceStartupTime.run.servicePort=90511 +#P2PRoundTripTime Metric +P2PRoundTripTime.enabled=true +P2PRoundTripTime.run.interval=100 +P2PRoundTripTime.run.sampleSize=5 +P2PRoundTripTime.run.hosts=http://723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 + #Another Metric Another.run.interval=5 diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java index 9db652bcd89..e171af66d1d 100644 --- a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java @@ -81,6 +81,7 @@ public void run(String sampleSize) throws Exception { configuration.put("P2PRoundTripTime.run.sampleSize", sampleSize); // torproject.org hidden service configuration.put("P2PRoundTripTime.run.hosts", "http://expyuzz4wqqyqhjn.onion:80"); + configuration.put("P2PRoundTripTime.run.torProxyPort", "9052"); Metric DUT = new P2PRoundTripTime(reporter); // start From 832bc4549a116ced02f80183e43a06eea3bebf19 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 21 Jan 2019 19:33:31 +0100 Subject: [PATCH 07/40] Set BTC_MAINNET for real --- monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java | 2 +- monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 566d70302c0..0d06db30637 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -62,7 +62,7 @@ public class P2PRoundTripTime extends Metric implements MessageListener, SetupLi public P2PRoundTripTime(Reporter reporter) { super(reporter); - Version.setBaseCryptoNetworkId(1); // set to BTC_MAINNET + Version.setBaseCryptoNetworkId(0); // set to BTC_MAINNET } @Override diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java index e171af66d1d..102d5cb7a27 100644 --- a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java @@ -80,7 +80,7 @@ public void run(String sampleSize) throws Exception { if (!"default".equals(sampleSize)) configuration.put("P2PRoundTripTime.run.sampleSize", sampleSize); // torproject.org hidden service - configuration.put("P2PRoundTripTime.run.hosts", "http://expyuzz4wqqyqhjn.onion:80"); + configuration.put("P2PRoundTripTime.run.hosts", "http://fl3mmribyxgrv63c.onion:8000"); configuration.put("P2PRoundTripTime.run.torProxyPort", "9052"); Metric DUT = new P2PRoundTripTime(reporter); From 6918f1bd271ada715e5af5b6ab01a7d8ef79419c Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 10:42:04 +0100 Subject: [PATCH 08/40] NetworkLoad Metric init --- .../bisq/monitor/metric/P2PNetworkLoad.java | 216 ++++++++++++++++++ .../bisq/monitor/P2PNetworkLoadTests.java | 112 +++++++++ 2 files changed, 328 insertions(+) create mode 100644 monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java create mode 100644 monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java new file mode 100644 index 00000000000..3a6b2a5f9d4 --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -0,0 +1,216 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor.metric; + +import java.io.File; +import java.net.URL; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.Set; + +import org.jetbrains.annotations.NotNull; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; + +import bisq.common.app.Version; +import bisq.common.proto.network.NetworkEnvelope; +import bisq.core.proto.network.CoreNetworkProtoResolver; +import bisq.monitor.Metric; +import bisq.monitor.Reporter; +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.network.Connection; +import bisq.network.p2p.network.MessageListener; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.network.NewTor; +import bisq.network.p2p.network.SetupListener; +import bisq.network.p2p.network.TorNetworkNode; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class P2PNetworkLoad extends Metric implements MessageListener, SetupListener { + + private static final String HOSTS = "run.hosts"; + private static final String TOR_PROXY_PORT = "run.torProxyPort"; + private NetworkNode networkNode; + private final File torWorkingDirectory = new File("metric_p2pNetworkLoad"); + private int nonce; + private Boolean ready = false; + + public P2PNetworkLoad(Reporter reporter) { + super(reporter); + + Version.setBaseCryptoNetworkId(0); // set to BTC_MAINNET + } + + @Override + protected void enable() { + Thread sepp = new Thread(new Runnable() { + + @Override + public void run() { + synchronized (ready) { + while (!ready) + try { + ready.wait(); + } catch (InterruptedException ignore) { + } + P2PNetworkLoad.super.enable(); + } + } + }); + sepp.start(); + } + + @Override + public void configure(Properties properties) { + super.configure(properties); + networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9053")), + new CoreNetworkProtoResolver(), false, + new NewTor(torWorkingDirectory, "", "", null)); + networkNode.start(this); + } + + /** + * synchronization helper. + */ + private void await() { + synchronized (torWorkingDirectory) { + try { + torWorkingDirectory.wait(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + private void proceed() { + synchronized (torWorkingDirectory) { + torWorkingDirectory.notify(); + } + } + + @Override + protected void execute() { + + // for each configured host + for (String current : configuration.getProperty(HOSTS, "").split(",")) { + try { + // parse Url + URL tmp = new URL(current); + NodeAddress target = new NodeAddress(tmp.getHost(), tmp.getPort()); + + nonce = new Random().nextInt(); + SettableFuture future = networkNode.sendMessage(target, + new PreliminaryGetDataRequest(nonce, new HashSet<>())); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Connection connection) { + connection.addMessageListener(P2PNetworkLoad.this); + log.debug("Send PreliminaryDataRequest to " + connection + " succeeded."); + } + + @Override + public void onFailure(@NotNull Throwable throwable) { + log.error( + "Sending PreliminaryDataRequest failed. That is expected if the peer is offline.\n\tException=" + + throwable.getMessage()); + } + }); + + await(); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + // report + } + + @Override + public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { + if (networkEnvelope instanceof GetDataResponse) { + + GetDataResponse getDataResponse = (GetDataResponse) networkEnvelope; + final Set dataSet = getDataResponse.getDataSet(); + dataSet.stream().forEach(e -> { + final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); + if (protectedStoragePayload == null) { + log.warn("StoragePayload was null: {}", networkEnvelope.toString()); + return; + } + + // memorize message hashes + // TODO cleanup hash list once in a while + + // For logging different data types + }); + + Set persistableNetworkPayloadSet = getDataResponse + .getPersistableNetworkPayloadSet(); + if (persistableNetworkPayloadSet != null) { + persistableNetworkPayloadSet.stream().forEach(persistableNetworkPayload -> { + + // memorize message hashes + // TODO cleanup hash list once in a while + + // For logging different data types + }); + } + + connection.removeMessageListener(this); + proceed(); + } + } + + @Override + public void onTorNodeReady() { + // TODO Auto-generated method stub + } + + @Override + public void onHiddenServicePublished() { + synchronized (ready) { + ready.notify(); + ready = true; + } + } + + @Override + public void onSetupFailed(Throwable throwable) { + // TODO Auto-generated method stub + + } + + @Override + public void onRequestCustomBridges() { + // TODO Auto-generated method stub + + } +} diff --git a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java new file mode 100644 index 00000000000..568459aeddf --- /dev/null +++ b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java @@ -0,0 +1,112 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import bisq.monitor.metric.P2PNetworkLoad; +import bisq.monitor.reporter.ConsoleReporter; + +import java.util.Map; +import java.util.Properties; + +import org.junit.Assert; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Test the round trip time metric against the hidden service of tor project.org. + * + * @author Florian Reimair + */ +public class P2PNetworkLoadTests { + + /** + * A dummy Reporter for development purposes. + */ + private class DummyReporter extends ConsoleReporter { + + private Map results; + + @Override + public void report(long value) { + Assert.fail(); + } + + public Map hasResults() { + return results; + } + + @Override + public void report(Map values) { + Assert.fail(); + } + + @Override + public void report(long value, String prefix) { + Assert.fail(); + } + + @Override + public void report(Map values, String prefix) { + super.report(values, prefix); + results = values; + } + } + + @ParameterizedTest + @ValueSource(strings = {"default", "3", "4", "10"}) + public void run(String sampleSize) throws Exception { + DummyReporter reporter = new DummyReporter(); + + // configure + Properties configuration = new Properties(); + configuration.put("P2PNetworkLoad.enabled", "true"); + configuration.put("P2PNetworkLoad.run.interval", "2"); + configuration.put("P2PNetworkLoad.run.hosts", "http://fl3mmribyxgrv63c.onion:8000"); + configuration.put("P2PNetworkLoad.run.torProxyPort", "9053"); + + Metric DUT = new P2PNetworkLoad(reporter); + // start + DUT.configure(configuration); + + // give it some time to start and then stop + while (!DUT.enabled()) + Thread.sleep(2000); + + DUT.shutdown(); + DUT.join(); + + // observe results + Map results = reporter.hasResults(); + Assert.assertFalse(results.isEmpty()); + Assert.assertEquals(results.get("sampleSize"), sampleSize.equals("default") ? "1" : sampleSize); + + Integer p25 = Integer.valueOf(results.get("p25")); + Integer p50 = Integer.valueOf(results.get("p50")); + Integer p75 = Integer.valueOf(results.get("p75")); + Integer min = Integer.valueOf(results.get("min")); + Integer max = Integer.valueOf(results.get("max")); + Integer average = Integer.valueOf(results.get("average")); + + Assert.assertTrue(0 < min); + Assert.assertTrue(min <= p25 && p25 <= p50); + Assert.assertTrue(p50 <= p75); + Assert.assertTrue(p75 <= max); + Assert.assertTrue(min <= average && average <= max); + } +} From 01924b838dd00c1a87c68fb016ebe8edc6567d02 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 10:49:02 +0100 Subject: [PATCH 09/40] Only request new data --- .../src/main/java/bisq/monitor/metric/P2PNetworkLoad.java | 6 +++++- monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 3a6b2a5f9d4..8a2244f61c8 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -45,6 +45,7 @@ import bisq.network.p2p.network.TorNetworkNode; import bisq.network.p2p.peers.getdata.messages.GetDataResponse; import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.P2PDataStorage; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStorageEntry; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; @@ -59,6 +60,7 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private final File torWorkingDirectory = new File("metric_p2pNetworkLoad"); private int nonce; private Boolean ready = false; + private Set hashes = new HashSet<>(); public P2PNetworkLoad(Reporter reporter) { super(reporter); @@ -126,7 +128,7 @@ protected void execute() { nonce = new Random().nextInt(); SettableFuture future = networkNode.sendMessage(target, - new PreliminaryGetDataRequest(nonce, new HashSet<>())); + new PreliminaryGetDataRequest(nonce, hashes)); Futures.addCallback(future, new FutureCallback() { @Override @@ -168,6 +170,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // memorize message hashes // TODO cleanup hash list once in a while + hashes.add(P2PDataStorage.get32ByteHash(protectedStoragePayload)); // For logging different data types }); @@ -179,6 +182,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // memorize message hashes // TODO cleanup hash list once in a while + hashes.add(persistableNetworkPayload.getHash()); // For logging different data types }); diff --git a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java index 568459aeddf..b9e207a7100 100644 --- a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java @@ -86,7 +86,7 @@ public void run(String sampleSize) throws Exception { // give it some time to start and then stop while (!DUT.enabled()) - Thread.sleep(2000); + Thread.sleep(100000); DUT.shutdown(); DUT.join(); From ed0dede26560c33ff65562274ab703fcaa709f3b Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 11:08:03 +0100 Subject: [PATCH 10/40] Collect NetworkLoad histogram --- .../bisq/monitor/metric/P2PNetworkLoad.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 8a2244f61c8..f6ab8a8f2b6 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -61,6 +61,23 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private int nonce; private Boolean ready = false; private Set hashes = new HashSet<>(); + private Map payloadByClassName; + + private class Counter { + private int value = 1; + + public int value() { + return value; + } + + public void increment() { + value++; + } + + public void clear() { + value = 0; + } + } public P2PNetworkLoad(Reporter reporter) { super(reporter); @@ -153,6 +170,10 @@ public void onFailure(@NotNull Throwable throwable) { } // report + Map report = new HashMap<>(); + payloadByClassName.forEach((type, counter) -> report.put(type, String.valueOf(counter.value()))); + + reporter.report(report, "bisq." + getName() + "." + "target"); } @Override @@ -160,6 +181,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof GetDataResponse) { GetDataResponse getDataResponse = (GetDataResponse) networkEnvelope; + payloadByClassName = new HashMap<>(); final Set dataSet = getDataResponse.getDataSet(); dataSet.stream().forEach(e -> { final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); @@ -173,6 +195,12 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { hashes.add(P2PDataStorage.get32ByteHash(protectedStoragePayload)); // For logging different data types + String className = protectedStoragePayload.getClass().getSimpleName(); + try { + payloadByClassName.get(className).increment(); + } catch (NullPointerException nullPointerException) { + payloadByClassName.put(className, new Counter()); + } }); Set persistableNetworkPayloadSet = getDataResponse @@ -185,6 +213,12 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { hashes.add(persistableNetworkPayload.getHash()); // For logging different data types + String className = persistableNetworkPayload.getClass().getSimpleName(); + try { + payloadByClassName.get(className).increment(); + } catch (NullPointerException nullPointerException) { + payloadByClassName.put(className, new Counter()); + } }); } From 724caf9fd1863d5940457ddbf4a2e447dbb9418f Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 13:41:17 +0100 Subject: [PATCH 11/40] Query multiple nodes for NetworkLoad metric --- .../bisq/monitor/metric/P2PNetworkLoad.java | 109 ++++++++++++------ .../bisq/monitor/P2PNetworkLoadTests.java | 30 ++--- 2 files changed, 79 insertions(+), 60 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index f6ab8a8f2b6..804b97a645f 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -19,12 +19,15 @@ import java.io.File; import java.net.URL; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Random; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import org.jetbrains.annotations.NotNull; import com.google.common.util.concurrent.FutureCallback; @@ -60,8 +63,9 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private final File torWorkingDirectory = new File("metric_p2pNetworkLoad"); private int nonce; private Boolean ready = false; - private Set hashes = new HashSet<>(); - private Map payloadByClassName; + private Map> bucketsPerHost = new ConcurrentHashMap<>(); + private CountDownLatch latch; + private Map> hashesPerHost = new ConcurrentHashMap<>();; private class Counter { private int value = 1; @@ -135,54 +139,76 @@ private void proceed() { @Override protected void execute() { + bucketsPerHost.clear(); + ArrayList threadList = new ArrayList<>(); // for each configured host for (String current : configuration.getProperty(HOSTS, "").split(",")) { - try { - // parse Url - URL tmp = new URL(current); - NodeAddress target = new NodeAddress(tmp.getHost(), tmp.getPort()); - - nonce = new Random().nextInt(); - SettableFuture future = networkNode.sendMessage(target, - new PreliminaryGetDataRequest(nonce, hashes)); - - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(Connection connection) { - connection.addMessageListener(P2PNetworkLoad.this); - log.debug("Send PreliminaryDataRequest to " + connection + " succeeded."); - } + threadList.add(new Thread(new Runnable() { - @Override - public void onFailure(@NotNull Throwable throwable) { - log.error( - "Sending PreliminaryDataRequest failed. That is expected if the peer is offline.\n\tException=" - + throwable.getMessage()); + @Override + public void run() { + try { + // parse Url + URL tmp = new URL(current); + NodeAddress target = new NodeAddress(tmp.getHost(), tmp.getPort()); + + nonce = new Random().nextInt(); + SettableFuture future = networkNode.sendMessage(target, + new PreliminaryGetDataRequest(nonce, + hashesPerHost.get(target.getFullAddress()) == null ? new HashSet<>() + : hashesPerHost.get(target.getFullAddress()))); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Connection connection) { + connection.addMessageListener(P2PNetworkLoad.this); + log.debug("Send PreliminaryDataRequest to " + connection + " succeeded."); + } + + @Override + public void onFailure(@NotNull Throwable throwable) { + log.error( + "Sending PreliminaryDataRequest failed. That is expected if the peer is offline.\n\tException=" + + throwable.getMessage()); + } + }); + + } catch (Exception e) { + e.printStackTrace(); } - }); - await(); + } + }, current)); + } - } catch (Exception e) { - e.printStackTrace(); - } + latch = new CountDownLatch(threadList.size()); + + threadList.forEach(Thread::start); + try { + latch.await(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } // report Map report = new HashMap<>(); - payloadByClassName.forEach((type, counter) -> report.put(type, String.valueOf(counter.value()))); + bucketsPerHost.forEach((host, buckets) -> buckets.forEach((type, counter) -> report.put(host.replace("http://", "").trim() + "." + type, + String.valueOf(counter.value())))); - reporter.report(report, "bisq." + getName() + "." + "target"); + reporter.report(report, "bisq." + getName()); } @Override public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof GetDataResponse) { - GetDataResponse getDataResponse = (GetDataResponse) networkEnvelope; - payloadByClassName = new HashMap<>(); - final Set dataSet = getDataResponse.getDataSet(); + + GetDataResponse dataResponse = (GetDataResponse) networkEnvelope; + Map buckets = new HashMap<>(); + Set hashes = new HashSet<>(); + final Set dataSet = dataResponse.getDataSet(); dataSet.stream().forEach(e -> { final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); if (protectedStoragePayload == null) { @@ -197,13 +223,13 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // For logging different data types String className = protectedStoragePayload.getClass().getSimpleName(); try { - payloadByClassName.get(className).increment(); + buckets.get(className).increment(); } catch (NullPointerException nullPointerException) { - payloadByClassName.put(className, new Counter()); + buckets.put(className, new Counter()); } }); - Set persistableNetworkPayloadSet = getDataResponse + Set persistableNetworkPayloadSet = dataResponse .getPersistableNetworkPayloadSet(); if (persistableNetworkPayloadSet != null) { persistableNetworkPayloadSet.stream().forEach(persistableNetworkPayload -> { @@ -215,15 +241,22 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // For logging different data types String className = persistableNetworkPayload.getClass().getSimpleName(); try { - payloadByClassName.get(className).increment(); + buckets.get(className).increment(); } catch (NullPointerException nullPointerException) { - payloadByClassName.put(className, new Counter()); + buckets.put(className, new Counter()); } }); } + try { + hashesPerHost.get(connection.peersNodeAddressProperty().getValue().getFullAddress()).addAll(hashes); + } catch (NullPointerException npe) { + hashesPerHost.put(connection.peersNodeAddressProperty().getValue().getFullAddress(), hashes); + } + bucketsPerHost.put(connection.peersNodeAddressProperty().getValue().getFullAddress(), buckets); + connection.removeMessageListener(this); - proceed(); + latch.countDown(); } } diff --git a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java index b9e207a7100..6381b1bd213 100644 --- a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java @@ -25,8 +25,7 @@ import org.junit.Assert; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.Test; /** * Test the round trip time metric against the hidden service of tor project.org. @@ -68,16 +67,16 @@ public void report(Map values, String prefix) { } } - @ParameterizedTest - @ValueSource(strings = {"default", "3", "4", "10"}) - public void run(String sampleSize) throws Exception { + @Test + public void run() throws Exception { DummyReporter reporter = new DummyReporter(); // configure Properties configuration = new Properties(); configuration.put("P2PNetworkLoad.enabled", "true"); - configuration.put("P2PNetworkLoad.run.interval", "2"); - configuration.put("P2PNetworkLoad.run.hosts", "http://fl3mmribyxgrv63c.onion:8000"); + configuration.put("P2PNetworkLoad.run.interval", "10"); + configuration.put("P2PNetworkLoad.run.hosts", + "http://fl3mmribyxgrv63c.onion:8000, http://3f3cu2yw7u457ztq.onion:8000"); configuration.put("P2PNetworkLoad.run.torProxyPort", "9053"); Metric DUT = new P2PNetworkLoad(reporter); @@ -86,7 +85,8 @@ public void run(String sampleSize) throws Exception { // give it some time to start and then stop while (!DUT.enabled()) - Thread.sleep(100000); + Thread.sleep(500); + Thread.sleep(20000); DUT.shutdown(); DUT.join(); @@ -94,19 +94,5 @@ public void run(String sampleSize) throws Exception { // observe results Map results = reporter.hasResults(); Assert.assertFalse(results.isEmpty()); - Assert.assertEquals(results.get("sampleSize"), sampleSize.equals("default") ? "1" : sampleSize); - - Integer p25 = Integer.valueOf(results.get("p25")); - Integer p50 = Integer.valueOf(results.get("p50")); - Integer p75 = Integer.valueOf(results.get("p75")); - Integer min = Integer.valueOf(results.get("min")); - Integer max = Integer.valueOf(results.get("max")); - Integer average = Integer.valueOf(results.get("average")); - - Assert.assertTrue(0 < min); - Assert.assertTrue(min <= p25 && p25 <= p50); - Assert.assertTrue(p50 <= p75); - Assert.assertTrue(p75 <= max); - Assert.assertTrue(min <= average && average <= max); } } From 662891c06295b7c3ccda8ff40b89b73bef34f712 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 13:41:46 +0100 Subject: [PATCH 12/40] Cleanup --- .../bisq/monitor/metric/P2PNetworkLoad.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 804b97a645f..b4e0ef77928 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -77,10 +77,6 @@ public int value() { public void increment() { value++; } - - public void clear() { - value = 0; - } } public P2PNetworkLoad(Reporter reporter) { @@ -117,26 +113,6 @@ public void configure(Properties properties) { networkNode.start(this); } - /** - * synchronization helper. - */ - private void await() { - synchronized (torWorkingDirectory) { - try { - torWorkingDirectory.wait(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - - private void proceed() { - synchronized (torWorkingDirectory) { - torWorkingDirectory.notify(); - } - } - @Override protected void execute() { bucketsPerHost.clear(); From 70d18e0565e5224b1badd5bc17db9949419adec8 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 14:25:11 +0100 Subject: [PATCH 13/40] Data consistency metrics --- .../bisq/monitor/metric/P2PNetworkLoad.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index b4e0ef77928..0637f6dbe97 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -23,11 +23,13 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; import org.jetbrains.annotations.NotNull; import com.google.common.util.concurrent.FutureCallback; @@ -170,9 +172,25 @@ public void onFailure(@NotNull Throwable throwable) { // report Map report = new HashMap<>(); + // - assemble histograms bucketsPerHost.forEach((host, buckets) -> buckets.forEach((type, counter) -> report.put(host.replace("http://", "").trim() + "." + type, String.valueOf(counter.value())))); + // - assemble diffs + Map messagesPerHost = new HashMap<>(); + bucketsPerHost.forEach((host, buckets) -> messagesPerHost.put(host, + buckets.values().stream().collect(Collectors.summingInt(Counter::value)))); + Optional referenceHost = messagesPerHost.keySet().stream().sorted().findFirst(); + Integer referenceValue = messagesPerHost.get(referenceHost.get()); + + messagesPerHost.forEach( + (host, numberOfMessages) -> { + report.put(host.replace("http://", "").trim() + ".relativeNumberOfMessages", + String.valueOf(numberOfMessages - referenceValue)); + report.put(host.replace("http://", "").trim() + ".referenceHost", referenceHost.get()); + report.put(host.replace("http://", "").trim() + ".referenceValue", String.valueOf(referenceValue)); + }); + reporter.report(report, "bisq." + getName()); } From 273b8c1692a14a89bad1c0412f7ec8e1b315789a Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 14:28:58 +0100 Subject: [PATCH 14/40] Updated sample config --- monitor/src/main/resources/metrics.properties | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/monitor/src/main/resources/metrics.properties b/monitor/src/main/resources/metrics.properties index 5e5a714014b..51c2e1683a2 100644 --- a/monitor/src/main/resources/metrics.properties +++ b/monitor/src/main/resources/metrics.properties @@ -29,6 +29,13 @@ P2PRoundTripTime.enabled=true P2PRoundTripTime.run.interval=100 P2PRoundTripTime.run.sampleSize=5 P2PRoundTripTime.run.hosts=http://723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 +P2PRoundTripTime.run.torProxyPort=90502 + +#P2PNetworkLoad Metric +P2PNetworkLoad.enabled=true +P2PNetworkLoad.run.interval=100 +P2PNetworkLoad.run.hosts=http://723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 +P2PNetworkLoad.run.torProxyPort=90503 #Another Metric Another.run.interval=5 From bafce455f88b7db2b4a751e40f90e9fbb95eff89 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 14:57:03 +0100 Subject: [PATCH 15/40] Limit hashlist in size --- .../bisq/monitor/metric/P2PNetworkLoad.java | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 0637f6dbe97..c9719d68c36 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -67,7 +67,8 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private Boolean ready = false; private Map> bucketsPerHost = new ConcurrentHashMap<>(); private CountDownLatch latch; - private Map> hashesPerHost = new ConcurrentHashMap<>();; + private Set hashes = new HashSet<>(); + private boolean reportFindings; private class Counter { private int value = 1; @@ -120,6 +121,14 @@ protected void execute() { bucketsPerHost.clear(); ArrayList threadList = new ArrayList<>(); + // in case we just started anew, do not report our findings as they contain not + // only the changes since our last run, but a whole lot more dating back even + // till the beginning of bisq. + if (hashes.isEmpty()) + reportFindings = false; + else + reportFindings = true; + // for each configured host for (String current : configuration.getProperty(HOSTS, "").split(",")) { threadList.add(new Thread(new Runnable() { @@ -133,9 +142,7 @@ public void run() { nonce = new Random().nextInt(); SettableFuture future = networkNode.sendMessage(target, - new PreliminaryGetDataRequest(nonce, - hashesPerHost.get(target.getFullAddress()) == null ? new HashSet<>() - : hashesPerHost.get(target.getFullAddress()))); + new PreliminaryGetDataRequest(nonce, hashes)); Futures.addCallback(future, new FutureCallback() { @Override @@ -191,7 +198,13 @@ public void onFailure(@NotNull Throwable throwable) { report.put(host.replace("http://", "").trim() + ".referenceValue", String.valueOf(referenceValue)); }); - reporter.report(report, "bisq." + getName()); + // when our hash cache exceeds a hard limit, we clear the cache and start anew + if (hashes.size() > 150000) + hashes.clear(); + + // report our findings iff we have not just started anew + if (reportFindings) + reporter.report(report, "bisq." + getName()); } @Override @@ -201,7 +214,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { GetDataResponse dataResponse = (GetDataResponse) networkEnvelope; Map buckets = new HashMap<>(); - Set hashes = new HashSet<>(); final Set dataSet = dataResponse.getDataSet(); dataSet.stream().forEach(e -> { final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); @@ -211,7 +223,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { } // memorize message hashes - // TODO cleanup hash list once in a while hashes.add(P2PDataStorage.get32ByteHash(protectedStoragePayload)); // For logging different data types @@ -229,7 +240,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { persistableNetworkPayloadSet.stream().forEach(persistableNetworkPayload -> { // memorize message hashes - // TODO cleanup hash list once in a while hashes.add(persistableNetworkPayload.getHash()); // For logging different data types @@ -242,11 +252,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { }); } - try { - hashesPerHost.get(connection.peersNodeAddressProperty().getValue().getFullAddress()).addAll(hashes); - } catch (NullPointerException npe) { - hashesPerHost.put(connection.peersNodeAddressProperty().getValue().getFullAddress(), hashes); - } bucketsPerHost.put(connection.peersNodeAddressProperty().getValue().getFullAddress(), buckets); connection.removeMessageListener(this); From d87038fbf8b13474f6abe0a4da36b25fa9d2a029 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 14:57:27 +0100 Subject: [PATCH 16/40] Comments --- .../bisq/monitor/metric/P2PNetworkLoad.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index c9719d68c36..c31b4b2e525 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -56,6 +56,15 @@ import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import lombok.extern.slf4j.Slf4j; +/** + * Contacts a list of hosts and asks them for all the data we do not have. The + * answers are then compiled into buckets of message types. Based on these + * buckets, the Metric reports (for each host) the message types observed and + * their number along with a relative comparison between all hosts. + * + * @author Florian Reimair + * + */ @Slf4j public class P2PNetworkLoad extends Metric implements MessageListener, SetupListener { @@ -70,6 +79,9 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private Set hashes = new HashSet<>(); private boolean reportFindings; + /** + * Efficient way to count message occurrences. + */ private class Counter { private int value = 1; @@ -118,6 +130,7 @@ public void configure(Properties properties) { @Override protected void execute() { + // clear our buckets bucketsPerHost.clear(); ArrayList threadList = new ArrayList<>(); @@ -140,6 +153,7 @@ public void run() { URL tmp = new URL(current); NodeAddress target = new NodeAddress(tmp.getHost(), tmp.getPort()); + // do the data request nonce = new Random().nextInt(); SettableFuture future = networkNode.sendMessage(target, new PreliminaryGetDataRequest(nonce, hashes)); @@ -169,6 +183,9 @@ public void onFailure(@NotNull Throwable throwable) { latch = new CountDownLatch(threadList.size()); + // start all threads and wait until they all finished. We do that so we can + // minimize the time between querying the hosts and therefore the chance of + // inconsistencies. threadList.forEach(Thread::start); try { latch.await(); From cae4a62341f4d6197664ac7742b7426d0dcc4f8b Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 24 Jan 2019 15:19:39 +0100 Subject: [PATCH 17/40] Use global Tor for P2P metrics --- .../main/java/bisq/monitor/AvailableTor.java | 52 +++++++++++++++++++ .../src/main/java/bisq/monitor/Monitor.java | 3 +- .../bisq/monitor/metric/P2PNetworkLoad.java | 7 +-- .../bisq/monitor/metric/P2PRoundTripTime.java | 15 +++--- .../bisq/monitor/metric/TorRoundTripTime.java | 5 -- .../bisq/monitor/P2PNetworkLoadTests.java | 21 +++++++- .../bisq/monitor/P2PRoundTripTimeTests.java | 20 +++++++ 7 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 monitor/src/main/java/bisq/monitor/AvailableTor.java diff --git a/monitor/src/main/java/bisq/monitor/AvailableTor.java b/monitor/src/main/java/bisq/monitor/AvailableTor.java new file mode 100644 index 00000000000..39041267e13 --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/AvailableTor.java @@ -0,0 +1,52 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import java.io.File; +import java.io.IOException; +import org.berndpruenster.netlayer.tor.Tor; +import org.berndpruenster.netlayer.tor.TorCtlException; +import bisq.network.p2p.network.TorMode; + +/** + * This class uses an already defined Tor via Tor.getDefault() + * + * @author Florian Reimair + * + */ +public class AvailableTor extends TorMode { + + private String hiddenServiceDirectory; + + public AvailableTor(File torWorkingDirectory, String hiddenServiceDirectory) { + super(torWorkingDirectory); + + this.hiddenServiceDirectory = hiddenServiceDirectory; + } + + @Override + public Tor getTor() throws IOException, TorCtlException { + return Tor.getDefault(); + } + + @Override + public String getHiddenServiceDirectory() { + return hiddenServiceDirectory; + } + +} diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java index 1b85e1553d8..7bb9eb0ea1a 100644 --- a/monitor/src/main/java/bisq/monitor/Monitor.java +++ b/monitor/src/main/java/bisq/monitor/Monitor.java @@ -48,6 +48,7 @@ @Slf4j public class Monitor { + public static final File TOR_WORKING_DIR = new File("monitor/monitor-tor"); private static String[] args = {}; public static void main(String[] args) throws Throwable { @@ -68,7 +69,7 @@ public static void main(String[] args) throws Throwable { private void start() throws Throwable { // start Tor - Tor.setDefault(new NativeTor(new File("monitor/monitor-tor"), null, null, false)); + Tor.setDefault(new NativeTor(TOR_WORKING_DIR, null, null, false)); // assemble Metrics // - create reporters diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index c31b4b2e525..3b57c11ec3b 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -39,13 +39,14 @@ import bisq.common.app.Version; import bisq.common.proto.network.NetworkEnvelope; import bisq.core.proto.network.CoreNetworkProtoResolver; +import bisq.monitor.AvailableTor; import bisq.monitor.Metric; +import bisq.monitor.Monitor; import bisq.monitor.Reporter; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; import bisq.network.p2p.network.NetworkNode; -import bisq.network.p2p.network.NewTor; import bisq.network.p2p.network.SetupListener; import bisq.network.p2p.network.TorNetworkNode; import bisq.network.p2p.peers.getdata.messages.GetDataResponse; @@ -71,7 +72,7 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private static final String HOSTS = "run.hosts"; private static final String TOR_PROXY_PORT = "run.torProxyPort"; private NetworkNode networkNode; - private final File torWorkingDirectory = new File("metric_p2pNetworkLoad"); + private final File torHiddenServiceDir = new File("metric_p2pNetworkLoad"); private int nonce; private Boolean ready = false; private Map> bucketsPerHost = new ConcurrentHashMap<>(); @@ -124,7 +125,7 @@ public void configure(Properties properties) { super.configure(properties); networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9053")), new CoreNetworkProtoResolver(), false, - new NewTor(torWorkingDirectory, "", "", null)); + new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); networkNode.start(this); } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 0d06db30637..4f79969f1e7 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -32,14 +32,15 @@ import bisq.common.app.Version; import bisq.common.proto.network.NetworkEnvelope; import bisq.core.proto.network.CoreNetworkProtoResolver; +import bisq.monitor.AvailableTor; import bisq.monitor.Metric; +import bisq.monitor.Monitor; import bisq.monitor.Reporter; import bisq.monitor.StatisticsHelper; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; import bisq.network.p2p.network.NetworkNode; -import bisq.network.p2p.network.NewTor; import bisq.network.p2p.network.SetupListener; import bisq.network.p2p.network.TorNetworkNode; import bisq.network.p2p.peers.keepalive.messages.Ping; @@ -53,7 +54,7 @@ public class P2PRoundTripTime extends Metric implements MessageListener, SetupLi private static final String HOSTS = "run.hosts"; private static final String TOR_PROXY_PORT = "run.torProxyPort"; private NetworkNode networkNode; - private final File torWorkingDirectory = new File("metric_p2pRoundTripTime"); + private final File torHiddenServiceDir = new File("metric_p2pRoundTripTime"); private int nonce; private long start; private Boolean ready = false; @@ -88,7 +89,7 @@ public void run() { public void configure(Properties properties) { super.configure(properties); networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9052")), new CoreNetworkProtoResolver(), false, - new NewTor(torWorkingDirectory, "", "", null)); + new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); networkNode.start(this); } @@ -96,9 +97,9 @@ public void configure(Properties properties) { * synchronization helper. */ private void await() { - synchronized (torWorkingDirectory) { + synchronized (torHiddenServiceDir) { try { - torWorkingDirectory.wait(); + torHiddenServiceDir.wait(); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -107,8 +108,8 @@ private void await() { } private void proceed() { - synchronized (torWorkingDirectory) { - torWorkingDirectory.notify(); + synchronized (torHiddenServiceDir) { + torHiddenServiceDir.notify(); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java index a0b94735a91..72e520301ee 100644 --- a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java @@ -32,12 +32,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.LongSummaryStatistics; -import java.util.Map; - import static com.google.common.base.Preconditions.checkNotNull; /** diff --git a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java index 6381b1bd213..5298402a2ec 100644 --- a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java @@ -20,10 +20,17 @@ import bisq.monitor.metric.P2PNetworkLoad; import bisq.monitor.reporter.ConsoleReporter; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.Map; import java.util.Properties; +import org.berndpruenster.netlayer.tor.NativeTor; +import org.berndpruenster.netlayer.tor.Tor; +import org.berndpruenster.netlayer.tor.TorCtlException; import org.junit.Assert; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -67,6 +74,12 @@ public void report(Map values, String prefix) { } } + @BeforeAll + public static void setup() throws TorCtlException { + // simulate the tor instance available to all metrics + Tor.setDefault(new NativeTor(Monitor.TOR_WORKING_DIR)); + } + @Test public void run() throws Exception { DummyReporter reporter = new DummyReporter(); @@ -77,7 +90,6 @@ public void run() throws Exception { configuration.put("P2PNetworkLoad.run.interval", "10"); configuration.put("P2PNetworkLoad.run.hosts", "http://fl3mmribyxgrv63c.onion:8000, http://3f3cu2yw7u457ztq.onion:8000"); - configuration.put("P2PNetworkLoad.run.torProxyPort", "9053"); Metric DUT = new P2PNetworkLoad(reporter); // start @@ -95,4 +107,11 @@ public void run() throws Exception { Map results = reporter.hasResults(); Assert.assertFalse(results.isEmpty()); } + + @AfterAll + public static void cleanup() { + Tor tor = Tor.getDefault(); + checkNotNull(tor, "tor must not be null"); + tor.shutdown(); + } } diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java index 102d5cb7a27..efb682917b5 100644 --- a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java @@ -20,10 +20,17 @@ import bisq.monitor.metric.P2PRoundTripTime; import bisq.monitor.reporter.ConsoleReporter; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.Map; import java.util.Properties; +import org.berndpruenster.netlayer.tor.NativeTor; +import org.berndpruenster.netlayer.tor.Tor; +import org.berndpruenster.netlayer.tor.TorCtlException; import org.junit.Assert; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -68,6 +75,12 @@ public void report(Map values, String prefix) { } } + @BeforeAll + public static void setup() throws TorCtlException { + // simulate the tor instance available to all metrics + Tor.setDefault(new NativeTor(Monitor.TOR_WORKING_DIR)); + } + @ParameterizedTest @ValueSource(strings = {"default", "3", "4", "10"}) public void run(String sampleSize) throws Exception { @@ -112,4 +125,11 @@ public void run(String sampleSize) throws Exception { Assert.assertTrue(p75 <= max); Assert.assertTrue(min <= average && average <= max); } + + @AfterAll + public static void cleanup() { + Tor tor = Tor.getDefault(); + checkNotNull(tor, "tor must not be null"); + tor.shutdown(); + } } From 2c3fcc6229c5a55eaa86b1c67854f83e759daf7f Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 28 Jan 2019 11:54:30 +0100 Subject: [PATCH 18/40] Added P2P metrics --- monitor/src/main/java/bisq/monitor/Monitor.java | 4 ++++ monitor/src/main/resources/metrics.properties | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java index 7bb9eb0ea1a..0f53e1e37d4 100644 --- a/monitor/src/main/java/bisq/monitor/Monitor.java +++ b/monitor/src/main/java/bisq/monitor/Monitor.java @@ -17,6 +17,8 @@ package bisq.monitor; +import bisq.monitor.metric.P2PNetworkLoad; +import bisq.monitor.metric.P2PRoundTripTime; import bisq.monitor.metric.TorHiddenServiceStartupTime; import bisq.monitor.metric.TorRoundTripTime; import bisq.monitor.metric.TorStartupTime; @@ -80,6 +82,8 @@ private void start() throws Throwable { metrics.add(new TorStartupTime(graphiteReporter)); metrics.add(new TorRoundTripTime(graphiteReporter)); metrics.add(new TorHiddenServiceStartupTime(graphiteReporter)); + metrics.add(new P2PRoundTripTime(graphiteReporter)); + metrics.add(new P2PNetworkLoad(graphiteReporter)); // prepare configuration reload // Note that this is most likely only work on Linux diff --git a/monitor/src/main/resources/metrics.properties b/monitor/src/main/resources/metrics.properties index 51c2e1683a2..83b12704931 100644 --- a/monitor/src/main/resources/metrics.properties +++ b/monitor/src/main/resources/metrics.properties @@ -29,13 +29,11 @@ P2PRoundTripTime.enabled=true P2PRoundTripTime.run.interval=100 P2PRoundTripTime.run.sampleSize=5 P2PRoundTripTime.run.hosts=http://723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 -P2PRoundTripTime.run.torProxyPort=90502 #P2PNetworkLoad Metric P2PNetworkLoad.enabled=true P2PNetworkLoad.run.interval=100 P2PNetworkLoad.run.hosts=http://723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 -P2PNetworkLoad.run.torProxyPort=90503 #Another Metric Another.run.interval=5 From 72075063fbd872a8c8f634134f539b08acc9cea7 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 28 Jan 2019 13:19:03 +0100 Subject: [PATCH 19/40] Refactored Onion address parsing --- .../main/java/bisq/monitor/OnionParser.java | 47 +++++++++++++++++++ .../bisq/monitor/metric/P2PNetworkLoad.java | 29 +++++++----- .../bisq/monitor/metric/P2PRoundTripTime.java | 8 ++-- .../bisq/monitor/metric/TorRoundTripTime.java | 8 ++-- .../monitor/reporter/GraphiteReporter.java | 9 ++-- 5 files changed, 76 insertions(+), 25 deletions(-) create mode 100644 monitor/src/main/java/bisq/monitor/OnionParser.java diff --git a/monitor/src/main/java/bisq/monitor/OnionParser.java b/monitor/src/main/java/bisq/monitor/OnionParser.java new file mode 100644 index 00000000000..53e1e5790db --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/OnionParser.java @@ -0,0 +1,47 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import java.net.MalformedURLException; +import java.net.URL; + +import bisq.network.p2p.NodeAddress; + +/** + * Helper for parsing and pretty printing onion addresses. + * + * @author Florian Reimair + */ +public class OnionParser { + + public static NodeAddress getNodeAddress(final String current) throws MalformedURLException { + String nodeAddress = current.trim(); + if (!nodeAddress.startsWith("http://")) + nodeAddress = "http://" + nodeAddress; + URL tmp = new URL(nodeAddress); + return new NodeAddress(tmp.getHost(), tmp.getPort()); + } + + public static String prettyPrint(final NodeAddress host) { + return host.getHostNameWithoutPostFix(); + } + + public static String prettyPrint(String host) throws MalformedURLException { + return prettyPrint(getNodeAddress(host)); + } +} diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 3b57c11ec3b..2ec94313cdb 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -18,7 +18,7 @@ package bisq.monitor.metric; import java.io.File; -import java.net.URL; +import java.net.MalformedURLException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -42,6 +42,7 @@ import bisq.monitor.AvailableTor; import bisq.monitor.Metric; import bisq.monitor.Monitor; +import bisq.monitor.OnionParser; import bisq.monitor.Reporter; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; @@ -75,7 +76,7 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private final File torHiddenServiceDir = new File("metric_p2pNetworkLoad"); private int nonce; private Boolean ready = false; - private Map> bucketsPerHost = new ConcurrentHashMap<>(); + private Map> bucketsPerHost = new ConcurrentHashMap<>(); private CountDownLatch latch; private Set hashes = new HashSet<>(); private boolean reportFindings; @@ -151,8 +152,7 @@ protected void execute() { public void run() { try { // parse Url - URL tmp = new URL(current); - NodeAddress target = new NodeAddress(tmp.getHost(), tmp.getPort()); + NodeAddress target = OnionParser.getNodeAddress(current); // do the data request nonce = new Random().nextInt(); @@ -198,22 +198,27 @@ public void onFailure(@NotNull Throwable throwable) { // report Map report = new HashMap<>(); // - assemble histograms - bucketsPerHost.forEach((host, buckets) -> buckets.forEach((type, counter) -> report.put(host.replace("http://", "").trim() + "." + type, - String.valueOf(counter.value())))); + bucketsPerHost.forEach((host, buckets) -> buckets.forEach((type, counter) -> report + .put(OnionParser.prettyPrint(host) + "." + type, String.valueOf(counter.value())))); // - assemble diffs Map messagesPerHost = new HashMap<>(); - bucketsPerHost.forEach((host, buckets) -> messagesPerHost.put(host, + bucketsPerHost.forEach((host, buckets) -> messagesPerHost.put(OnionParser.prettyPrint(host), buckets.values().stream().collect(Collectors.summingInt(Counter::value)))); Optional referenceHost = messagesPerHost.keySet().stream().sorted().findFirst(); Integer referenceValue = messagesPerHost.get(referenceHost.get()); messagesPerHost.forEach( (host, numberOfMessages) -> { - report.put(host.replace("http://", "").trim() + ".relativeNumberOfMessages", - String.valueOf(numberOfMessages - referenceValue)); - report.put(host.replace("http://", "").trim() + ".referenceHost", referenceHost.get()); - report.put(host.replace("http://", "").trim() + ".referenceValue", String.valueOf(referenceValue)); + try { + report.put(OnionParser.prettyPrint(host) + ".relativeNumberOfMessages", + String.valueOf(numberOfMessages - referenceValue)); + report.put(OnionParser.prettyPrint(host) + ".referenceHost", referenceHost.get()); + report.put(OnionParser.prettyPrint(host) + ".referenceValue", String.valueOf(referenceValue)); + } catch (MalformedURLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } }); // when our hash cache exceeds a hard limit, we clear the cache and start anew @@ -270,7 +275,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { }); } - bucketsPerHost.put(connection.peersNodeAddressProperty().getValue().getFullAddress(), buckets); + bucketsPerHost.put(connection.peersNodeAddressProperty().getValue(), buckets); connection.removeMessageListener(this); latch.countDown(); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 4f79969f1e7..5e07a83f605 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -18,7 +18,6 @@ package bisq.monitor.metric; import java.io.File; -import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.Properties; @@ -35,6 +34,7 @@ import bisq.monitor.AvailableTor; import bisq.monitor.Metric; import bisq.monitor.Monitor; +import bisq.monitor.OnionParser; import bisq.monitor.Reporter; import bisq.monitor.StatisticsHelper; import bisq.network.p2p.NodeAddress; @@ -120,8 +120,7 @@ protected void execute() { for (String current : configuration.getProperty(HOSTS, "").split(",")) { try { // parse Url - URL tmp = new URL(current); - NodeAddress target = new NodeAddress(tmp.getHost(), tmp.getPort()); + NodeAddress target = OnionParser.getNodeAddress(current); // init sample bucket samples = new ArrayList<>(); @@ -149,7 +148,8 @@ public void onFailure(@NotNull Throwable throwable) { } // report - reporter.report(StatisticsHelper.process(samples), "bisq." + getName() + "." + target); + reporter.report(StatisticsHelper.process(samples), + "bisq." + getName() + "." + OnionParser.prettyPrint(target)); } catch (Exception e) { e.printStackTrace(); } diff --git a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java index 72e520301ee..5616529ed93 100644 --- a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java @@ -18,8 +18,10 @@ package bisq.monitor.metric; import bisq.monitor.Metric; +import bisq.monitor.OnionParser; import bisq.monitor.Reporter; import bisq.monitor.StatisticsHelper; +import bisq.network.p2p.NodeAddress; import org.berndpruenster.netlayer.tor.Tor; import org.berndpruenster.netlayer.tor.TorCtlException; @@ -27,8 +29,6 @@ import com.runjva.sourceforge.jsocks.protocol.Socks5Proxy; import com.runjva.sourceforge.jsocks.protocol.SocksSocket; -import java.net.URL; - import java.io.IOException; import java.util.ArrayList; @@ -61,7 +61,7 @@ protected void execute() { // for each configured host for (String current : configuration.getProperty(HOSTS, "").split(",")) { // parse Url - URL tmp = new URL(current); + NodeAddress tmp = OnionParser.getNodeAddress(current); List samples = new ArrayList<>(); @@ -71,7 +71,7 @@ protected void execute() { long start = System.currentTimeMillis(); // connect - socket = new SocksSocket(proxy, tmp.getHost(), tmp.getPort()); + socket = new SocksSocket(proxy, tmp.getHostName(), tmp.getPort()); // by the time we get here, we are connected samples.add(System.currentTimeMillis() - start); diff --git a/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java b/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java index fd18f3af9ce..890075c75a6 100644 --- a/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java +++ b/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java @@ -17,12 +17,12 @@ package bisq.monitor.reporter; +import bisq.monitor.OnionParser; import bisq.monitor.Reporter; +import bisq.network.p2p.NodeAddress; import org.berndpruenster.netlayer.tor.TorSocket; -import java.net.URL; - import java.io.IOException; import java.util.HashMap; @@ -55,10 +55,9 @@ public void report(Map values, String prefix) { String report = prefix + ("".equals(key) ? "" : (prefix.isEmpty() ? "" : ".") + key) + " " + value + " " + timestamp + "\n"; - URL url; try { - url = new URL(configuration.getProperty("serviceUrl")); - TorSocket socket = new TorSocket(url.getHost(), url.getPort()); + NodeAddress nodeAddress = OnionParser.getNodeAddress(configuration.getProperty("serviceUrl")); + TorSocket socket = new TorSocket(nodeAddress.getHostName(), nodeAddress.getPort()); socket.getOutputStream().write(report.getBytes()); socket.close(); From 06876dd0a6cd13f11ecd1a961925727144139485 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Mon, 28 Jan 2019 19:47:18 +0100 Subject: [PATCH 20/40] Only start network nodes if metric is enabled --- .../bisq/monitor/metric/P2PNetworkLoad.java | 46 ++++++++----------- .../bisq/monitor/metric/P2PRoundTripTime.java | 38 +++++++-------- 2 files changed, 35 insertions(+), 49 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 2ec94313cdb..59ced7c47df 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -103,35 +103,26 @@ public P2PNetworkLoad(Reporter reporter) { } @Override - protected void enable() { - Thread sepp = new Thread(new Runnable() { - - @Override - public void run() { - synchronized (ready) { - while (!ready) - try { - ready.wait(); - } catch (InterruptedException ignore) { - } - P2PNetworkLoad.super.enable(); - } + protected void execute() { + // in case we do not have a NetworkNode up and running, we create one + if (null == networkNode) { + // prepare the gate + + networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9053")), + new CoreNetworkProtoResolver(), false, + new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); + networkNode.start(this); + + // wait for the HS to be published + synchronized (ready) { + while (!ready) + try { + ready.wait(); + } catch (InterruptedException ignore) { + } } - }); - sepp.start(); - } - - @Override - public void configure(Properties properties) { - super.configure(properties); - networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9053")), - new CoreNetworkProtoResolver(), false, - new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); - networkNode.start(this); - } + } - @Override - protected void execute() { // clear our buckets bucketsPerHost.clear(); ArrayList threadList = new ArrayList<>(); @@ -289,6 +280,7 @@ public void onTorNodeReady() { @Override public void onHiddenServicePublished() { + // open the gate synchronized (ready) { ready.notify(); ready = true; diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 5e07a83f605..3d1a6e9e2b4 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -66,31 +66,9 @@ public P2PRoundTripTime(Reporter reporter) { Version.setBaseCryptoNetworkId(0); // set to BTC_MAINNET } - @Override - protected void enable() { - Thread sepp = new Thread(new Runnable() { - - @Override - public void run() { - synchronized (ready) { - while (!ready) - try { - ready.wait(); - } catch (InterruptedException ignore) { - } - P2PRoundTripTime.super.enable(); - } - } - }); - sepp.start(); - } - @Override public void configure(Properties properties) { super.configure(properties); - networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9052")), new CoreNetworkProtoResolver(), false, - new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); - networkNode.start(this); } /** @@ -116,6 +94,22 @@ private void proceed() { @Override protected void execute() { + if (null == networkNode) { + networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9052")), + new CoreNetworkProtoResolver(), false, + new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); + networkNode.start(this); + + synchronized (ready) { + while (!ready) + try { + ready.wait(); + } catch (InterruptedException ignore) { + } + } + } + + // for each configured host for (String current : configuration.getProperty(HOSTS, "").split(",")) { try { From e5f4776ea61dfedcb435534eb0df7022f6957a11 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Tue, 29 Jan 2019 12:42:20 +0100 Subject: [PATCH 21/40] Refactored thread synchronization --- .../main/java/bisq/monitor/ThreadGate.java | 56 +++++++++++++++++++ .../bisq/monitor/metric/P2PNetworkLoad.java | 18 ++---- .../bisq/monitor/metric/P2PRoundTripTime.java | 52 ++++++----------- .../metric/TorHiddenServiceStartupTime.java | 30 +++------- 4 files changed, 87 insertions(+), 69 deletions(-) create mode 100644 monitor/src/main/java/bisq/monitor/ThreadGate.java diff --git a/monitor/src/main/java/bisq/monitor/ThreadGate.java b/monitor/src/main/java/bisq/monitor/ThreadGate.java new file mode 100644 index 00000000000..5a8e6937be9 --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/ThreadGate.java @@ -0,0 +1,56 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import java.util.concurrent.CountDownLatch; + +/** + * Gate pattern to help with thread synchronization + * + * @author Florian Reimair + */ +public class ThreadGate { + + private CountDownLatch lock = new CountDownLatch(0); + + /** + * Make everyone wait until the gate is opened again. + */ + public void engage() { + lock = new CountDownLatch(1); + } + + /** + * Wait for the gate to be opened. Blocks until the gate is opened again. + * Returns immediately if the gate is already open. + */ + public synchronized void await() { + while (lock.getCount() > 0) + try { + lock.await(); + } catch (InterruptedException ignore) { + } + } + + /** + * Open the gate and let everyone proceed with their execution. + */ + public void proceed() { + lock.countDown(); + } +} diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 59ced7c47df..d2ffcc6a536 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -44,6 +44,7 @@ import bisq.monitor.Monitor; import bisq.monitor.OnionParser; import bisq.monitor.Reporter; +import bisq.monitor.ThreadGate; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; @@ -75,11 +76,11 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private NetworkNode networkNode; private final File torHiddenServiceDir = new File("metric_p2pNetworkLoad"); private int nonce; - private Boolean ready = false; private Map> bucketsPerHost = new ConcurrentHashMap<>(); private CountDownLatch latch; private Set hashes = new HashSet<>(); private boolean reportFindings; + private final ThreadGate hsReady = new ThreadGate(); /** * Efficient way to count message occurrences. @@ -107,20 +108,16 @@ protected void execute() { // in case we do not have a NetworkNode up and running, we create one if (null == networkNode) { // prepare the gate + hsReady.engage(); + // start the network node networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9053")), new CoreNetworkProtoResolver(), false, new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); networkNode.start(this); // wait for the HS to be published - synchronized (ready) { - while (!ready) - try { - ready.wait(); - } catch (InterruptedException ignore) { - } - } + hsReady.await(); } // clear our buckets @@ -281,10 +278,7 @@ public void onTorNodeReady() { @Override public void onHiddenServicePublished() { // open the gate - synchronized (ready) { - ready.notify(); - ready = true; - } + hsReady.proceed(); } @Override diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 3d1a6e9e2b4..4a3def57bd1 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -37,6 +37,7 @@ import bisq.monitor.OnionParser; import bisq.monitor.Reporter; import bisq.monitor.StatisticsHelper; +import bisq.monitor.ThreadGate; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; @@ -57,8 +58,9 @@ public class P2PRoundTripTime extends Metric implements MessageListener, SetupLi private final File torHiddenServiceDir = new File("metric_p2pRoundTripTime"); private int nonce; private long start; - private Boolean ready = false; private List samples; + private final ThreadGate gate = new ThreadGate(); + private final ThreadGate hsReady = new ThreadGate(); public P2PRoundTripTime(Reporter reporter) { super(reporter); @@ -71,42 +73,20 @@ public void configure(Properties properties) { super.configure(properties); } - /** - * synchronization helper. - */ - private void await() { - synchronized (torHiddenServiceDir) { - try { - torHiddenServiceDir.wait(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - - private void proceed() { - synchronized (torHiddenServiceDir) { - torHiddenServiceDir.notify(); - } - } - @Override protected void execute() { if (null == networkNode) { + // close the gate + hsReady.engage(); + networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9052")), new CoreNetworkProtoResolver(), false, new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); networkNode.start(this); - synchronized (ready) { - while (!ready) - try { - ready.wait(); - } catch (InterruptedException ignore) { - } - } + // wait for the gate to be reopened + hsReady.await(); } @@ -121,6 +101,10 @@ protected void execute() { while (samples.size() < Integer.parseInt(configuration.getProperty(SAMPLE_SIZE, "1"))) { nonce = new Random().nextInt(); + + // close the gate + gate.engage(); + start = System.currentTimeMillis(); SettableFuture future = networkNode.sendMessage(target, new Ping(nonce, 42)); @@ -138,7 +122,8 @@ public void onFailure(@NotNull Throwable throwable) { } }); - await(); + // wait for the gate to open again + gate.await(); } // report @@ -162,7 +147,9 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { nonce, pong.getRequestNonce()); } connection.removeMessageListener(this); - proceed(); + + // open the gate + gate.proceed(); } } @@ -173,10 +160,7 @@ public void onTorNodeReady() { @Override public void onHiddenServicePublished() { - synchronized (ready) { - ready.notify(); - ready = true; - } + hsReady.proceed(); } @Override diff --git a/monitor/src/main/java/bisq/monitor/metric/TorHiddenServiceStartupTime.java b/monitor/src/main/java/bisq/monitor/metric/TorHiddenServiceStartupTime.java index 39ba8bd75fb..5ef02604df3 100644 --- a/monitor/src/main/java/bisq/monitor/metric/TorHiddenServiceStartupTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/TorHiddenServiceStartupTime.java @@ -19,6 +19,7 @@ import bisq.monitor.Metric; import bisq.monitor.Reporter; +import bisq.monitor.ThreadGate; import org.berndpruenster.netlayer.tor.HiddenServiceSocket; @@ -38,32 +39,12 @@ public class TorHiddenServiceStartupTime extends Metric { private static final String SERVICE_PORT = "run.servicePort"; private static final String LOCAL_PORT = "run.localPort"; private final String hiddenServiceDirectory = "metric_" + getName(); + private final ThreadGate gate = new ThreadGate(); public TorHiddenServiceStartupTime(Reporter reporter) { super(reporter); } - /** - * synchronization helper. Required because directly closing the - * HiddenServiceSocket in its ReadyListener causes a deadlock - */ - private void await() { - synchronized (hiddenServiceDirectory) { - try { - hiddenServiceDirectory.wait(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - - private void proceed() { - synchronized (hiddenServiceDirectory) { - hiddenServiceDirectory.notify(); - } - } - @Override protected void execute() { // prepare settings. Fetch them every time we run the Metric so we do not have to @@ -75,6 +56,9 @@ protected void execute() { new File(hiddenServiceDirectory).delete(); log.debug("creating the hidden service"); + + gate.engage(); + // start timer - we do not need System.nanoTime as we expect our result to be in // the range of tenth of seconds. long start = System.currentTimeMillis(); @@ -85,11 +69,11 @@ protected void execute() { // stop the timer and report reporter.report(System.currentTimeMillis() - start, "bisq." + getName()); log.debug("the hidden service is ready"); - proceed(); + gate.proceed(); return null; }); - await(); + gate.await(); log.debug("going to revoke the hidden service..."); hiddenServiceSocket.close(); log.debug("[going to revoke the hidden service...] done"); From 2aaa81b00316324e3bb5d0e31992d5f14300ad8a Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Tue, 29 Jan 2019 12:44:10 +0100 Subject: [PATCH 22/40] Shutdown metric before sleep if shut down --- monitor/src/main/java/bisq/monitor/Metric.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/Metric.java b/monitor/src/main/java/bisq/monitor/Metric.java index 7838a3b7a38..53a2fcd7f37 100644 --- a/monitor/src/main/java/bisq/monitor/Metric.java +++ b/monitor/src/main/java/bisq/monitor/Metric.java @@ -117,9 +117,14 @@ public void run() { while (!shutdown) { // if not, execute all the things synchronized (this) { + log.info("{} started", getName()); execute(); + log.info("{} done", getName()); } + if (shutdown) + continue; + // make sure our configuration is not changed in the moment we want to query it String interval; synchronized (this) { From 202acac74c8ab312b8526c48f535aa4a2ccc1bb0 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Tue, 29 Jan 2019 12:55:46 +0100 Subject: [PATCH 23/40] Refactored lock implementation --- .../src/main/java/bisq/monitor/ThreadGate.java | 16 +++++++++++++--- .../java/bisq/monitor/metric/P2PNetworkLoad.java | 15 ++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/ThreadGate.java b/monitor/src/main/java/bisq/monitor/ThreadGate.java index 5a8e6937be9..73897e203c4 100644 --- a/monitor/src/main/java/bisq/monitor/ThreadGate.java +++ b/monitor/src/main/java/bisq/monitor/ThreadGate.java @@ -29,15 +29,25 @@ public class ThreadGate { private CountDownLatch lock = new CountDownLatch(0); /** - * Make everyone wait until the gate is opened again. + * Make everyone wait until the gate is open again. */ public void engage() { lock = new CountDownLatch(1); } /** - * Wait for the gate to be opened. Blocks until the gate is opened again. - * Returns immediately if the gate is already open. + * Make everyone wait until the gate is open again. + * + * @param numberOfLocks how often the gate has to be unlocked until the gate + * opens. + */ + public void engage(int numberOfLocks) { + lock = new CountDownLatch(numberOfLocks); + } + + /** + * Wait for the gate to be opened. Blocks until the gate is open again. Returns + * immediately if the gate is already open. */ public synchronized void await() { while (lock.getCount() > 0) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index d2ffcc6a536..c030fb8e68f 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -24,11 +24,9 @@ import java.util.HashSet; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; import org.jetbrains.annotations.NotNull; @@ -77,10 +75,10 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private final File torHiddenServiceDir = new File("metric_p2pNetworkLoad"); private int nonce; private Map> bucketsPerHost = new ConcurrentHashMap<>(); - private CountDownLatch latch; private Set hashes = new HashSet<>(); private boolean reportFindings; private final ThreadGate hsReady = new ThreadGate(); + private final ThreadGate gate = new ThreadGate(); /** * Efficient way to count message occurrences. @@ -170,18 +168,13 @@ public void onFailure(@NotNull Throwable throwable) { }, current)); } - latch = new CountDownLatch(threadList.size()); + gate.engage(threadList.size()); // start all threads and wait until they all finished. We do that so we can // minimize the time between querying the hosts and therefore the chance of // inconsistencies. threadList.forEach(Thread::start); - try { - latch.await(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + gate.await(); // report Map report = new HashMap<>(); @@ -266,7 +259,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { bucketsPerHost.put(connection.peersNodeAddressProperty().getValue(), buckets); connection.removeMessageListener(this); - latch.countDown(); + gate.proceed(); } } From 692da95017831401599ba7dec72438a97c5eaf89 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Tue, 29 Jan 2019 17:09:59 +0100 Subject: [PATCH 24/40] Unlock gate in case of an error --- monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java | 1 + monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java | 1 + 2 files changed, 2 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index c030fb8e68f..9abf5c9e189 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -161,6 +161,7 @@ public void onFailure(@NotNull Throwable throwable) { }); } catch (Exception e) { + gate.proceed(); // release the gate on error e.printStackTrace(); } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 4a3def57bd1..725fd706e7a 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -130,6 +130,7 @@ public void onFailure(@NotNull Throwable throwable) { reporter.report(StatisticsHelper.process(samples), "bisq." + getName() + "." + OnionParser.prettyPrint(target)); } catch (Exception e) { + gate.proceed(); // release the gate on error e.printStackTrace(); } } From eeca2d3e5fab00fda178b89c511c3648775d2ef3 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Tue, 29 Jan 2019 17:52:26 +0100 Subject: [PATCH 25/40] Added P2P Message Snapshot metric --- .../src/main/java/bisq/monitor/Monitor.java | 2 + .../metric/P2PNetworkMessageSnapshot.java | 253 ++++++++++++++++++ monitor/src/main/resources/metrics.properties | 12 +- 3 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java index 0f53e1e37d4..ad69bb4aef5 100644 --- a/monitor/src/main/java/bisq/monitor/Monitor.java +++ b/monitor/src/main/java/bisq/monitor/Monitor.java @@ -18,6 +18,7 @@ package bisq.monitor; import bisq.monitor.metric.P2PNetworkLoad; +import bisq.monitor.metric.P2PNetworkMessageSnapshot; import bisq.monitor.metric.P2PRoundTripTime; import bisq.monitor.metric.TorHiddenServiceStartupTime; import bisq.monitor.metric.TorRoundTripTime; @@ -84,6 +85,7 @@ private void start() throws Throwable { metrics.add(new TorHiddenServiceStartupTime(graphiteReporter)); metrics.add(new P2PRoundTripTime(graphiteReporter)); metrics.add(new P2PNetworkLoad(graphiteReporter)); + metrics.add(new P2PNetworkMessageSnapshot(graphiteReporter)); // prepare configuration reload // Note that this is most likely only work on Linux diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java new file mode 100644 index 00000000000..4658349618f --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java @@ -0,0 +1,253 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor.metric; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.jetbrains.annotations.NotNull; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; + +import bisq.common.app.Version; +import bisq.common.proto.network.NetworkEnvelope; +import bisq.core.proto.network.CoreNetworkProtoResolver; +import bisq.monitor.AvailableTor; +import bisq.monitor.Metric; +import bisq.monitor.Monitor; +import bisq.monitor.OnionParser; +import bisq.monitor.Reporter; +import bisq.monitor.ThreadGate; +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.network.Connection; +import bisq.network.p2p.network.MessageListener; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.network.SetupListener; +import bisq.network.p2p.network.TorNetworkNode; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; +import lombok.extern.slf4j.Slf4j; + +/** + * Contacts a list of hosts and asks them for all the data we do not have. The + * answers are then compiled into buckets of message types. Based on these + * buckets, the Metric reports (for each host) the message types observed and + * their number along with a relative comparison between all hosts. + * + * @author Florian Reimair + * + */ +@Slf4j +public class P2PNetworkMessageSnapshot extends Metric implements MessageListener, SetupListener { + + private static final String HOSTS = "run.hosts"; + private static final String TOR_PROXY_PORT = "run.torProxyPort"; + private NetworkNode networkNode; + private final File torHiddenServiceDir = new File("metric_p2pNetworkMessageStatus"); + private int nonce; + private Map> bucketsPerHost = new ConcurrentHashMap<>(); + private Set hashes = new HashSet<>(); + private boolean reportFindings; + private final ThreadGate hsReady = new ThreadGate(); + private final ThreadGate gate = new ThreadGate(); + + /** + * Efficient way to count message occurrences. + */ + private class Counter { + private int value = 1; + + public int value() { + return value; + } + + public void increment() { + value++; + } + } + + public P2PNetworkMessageSnapshot(Reporter reporter) { + super(reporter); + + Version.setBaseCryptoNetworkId(0); // set to BTC_MAINNET + } + + @Override + protected void execute() { + // in case we do not have a NetworkNode up and running, we create one + if (null == networkNode) { + // prepare the gate + hsReady.engage(); + + // start the network node + networkNode = new TorNetworkNode(Integer.parseInt(configuration.getProperty(TOR_PROXY_PORT, "9054")), + new CoreNetworkProtoResolver(), false, + new AvailableTor(Monitor.TOR_WORKING_DIR, torHiddenServiceDir.getName())); + networkNode.start(this); + + // wait for the HS to be published + hsReady.await(); + } + + // clear our buckets + bucketsPerHost.clear(); + ArrayList threadList = new ArrayList<>(); + + // in case we just started anew, do not report our findings as they contain not + // only the changes since our last run, but a whole lot more dating back even + // till the beginning of bisq. + if (hashes.isEmpty()) + reportFindings = false; + else + reportFindings = true; + + // for each configured host + for (String current : configuration.getProperty(HOSTS, "").split(",")) { + threadList.add(new Thread(new Runnable() { + + @Override + public void run() { + try { + // parse Url + NodeAddress target = OnionParser.getNodeAddress(current); + + // do the data request + nonce = new Random().nextInt(); + SettableFuture future = networkNode.sendMessage(target, + new PreliminaryGetDataRequest(nonce, hashes)); + + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Connection connection) { + connection.addMessageListener(P2PNetworkMessageSnapshot.this); + log.debug("Send PreliminaryDataRequest to " + connection + " succeeded."); + } + + @Override + public void onFailure(@NotNull Throwable throwable) { + log.error( + "Sending PreliminaryDataRequest failed. That is expected if the peer is offline.\n\tException=" + + throwable.getMessage()); + } + }); + + } catch (Exception e) { + gate.proceed(); // release the gate on error + e.printStackTrace(); + } + + } + }, current)); + } + + gate.engage(threadList.size()); + + // start all threads and wait until they all finished. We do that so we can + // minimize the time between querying the hosts and therefore the chance of + // inconsistencies. + threadList.forEach(Thread::start); + gate.await(); + + // report + Map report = new HashMap<>(); + // - assemble histograms + bucketsPerHost.forEach((host, buckets) -> buckets.forEach((type, counter) -> report + .put(OnionParser.prettyPrint(host) + "." + type, String.valueOf(counter.value())))); + + // when our hash cache exceeds a hard limit, we clear the cache and start anew + if (hashes.size() > 150000) + hashes.clear(); + + // report our findings iff we have not just started anew + if (reportFindings) + reporter.report(report, "bisq." + getName()); + } + + @Override + public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { + if (networkEnvelope instanceof GetDataResponse) { + + + GetDataResponse dataResponse = (GetDataResponse) networkEnvelope; + Map buckets = new HashMap<>(); + final Set dataSet = dataResponse.getDataSet(); + dataSet.stream().forEach(e -> { + final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); + if (protectedStoragePayload == null) { + log.warn("StoragePayload was null: {}", networkEnvelope.toString()); + return; + } + + // For logging different data types + String className = protectedStoragePayload.getClass().getSimpleName(); + try { + buckets.get(className).increment(); + } catch (NullPointerException nullPointerException) { + buckets.put(className, new Counter()); + } + }); + + Set persistableNetworkPayloadSet = dataResponse + .getPersistableNetworkPayloadSet(); + if (persistableNetworkPayloadSet != null) { + persistableNetworkPayloadSet.stream().forEach(persistableNetworkPayload -> { + + // memorize message hashes + hashes.add(persistableNetworkPayload.getHash()); + }); + } + + bucketsPerHost.put(connection.peersNodeAddressProperty().getValue(), buckets); + + connection.removeMessageListener(this); + gate.proceed(); + } + } + + @Override + public void onTorNodeReady() { + // TODO Auto-generated method stub + } + + @Override + public void onHiddenServicePublished() { + // open the gate + hsReady.proceed(); + } + + @Override + public void onSetupFailed(Throwable throwable) { + // TODO Auto-generated method stub + + } + + @Override + public void onRequestCustomBridges() { + // TODO Auto-generated method stub + + } +} diff --git a/monitor/src/main/resources/metrics.properties b/monitor/src/main/resources/metrics.properties index 83b12704931..bc5c662a30e 100644 --- a/monitor/src/main/resources/metrics.properties +++ b/monitor/src/main/resources/metrics.properties @@ -28,12 +28,18 @@ TorHiddenServiceStartupTime.run.servicePort=90511 P2PRoundTripTime.enabled=true P2PRoundTripTime.run.interval=100 P2PRoundTripTime.run.sampleSize=5 -P2PRoundTripTime.run.hosts=http://723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 +P2PRoundTripTime.run.hosts=723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 #P2PNetworkLoad Metric P2PNetworkLoad.enabled=true P2PNetworkLoad.run.interval=100 -P2PNetworkLoad.run.hosts=http://723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 +P2PNetworkLoad.run.hosts=723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 + +#P2PNetworkMessageSnapshot Metric +P2PNetworkMessageSnapshot.enabled=true +P2PNetworkMessageSnapshot.run.interval=24 +P2PNetworkMessageSnapshot.run.hosts=3f3cu2yw7u457ztq.onion:8000, 723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 + #Another Metric Another.run.interval=5 @@ -43,4 +49,4 @@ Another.run.interval=5 ## In contrast to Metrics, Reporters do not have a minimal set of properties. #GraphiteReporter -GraphiteReporter.serviceUrl=http://k6evlhg44acpchtc.onion:2003 +GraphiteReporter.serviceUrl=k6evlhg44acpchtc.onion:2003 From 7c6e71cc720d1b91a11ee99a4426c339be5ddc82 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Tue, 29 Jan 2019 18:10:59 +0100 Subject: [PATCH 26/40] Unlock gates in case of connection shutdown --- monitor/src/main/java/bisq/monitor/ThreadGate.java | 8 ++++++++ .../src/main/java/bisq/monitor/metric/P2PNetworkLoad.java | 6 ++++++ .../bisq/monitor/metric/P2PNetworkMessageSnapshot.java | 6 ++++++ .../main/java/bisq/monitor/metric/P2PRoundTripTime.java | 5 +++++ 4 files changed, 25 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/ThreadGate.java b/monitor/src/main/java/bisq/monitor/ThreadGate.java index 73897e203c4..46c0d4e6ade 100644 --- a/monitor/src/main/java/bisq/monitor/ThreadGate.java +++ b/monitor/src/main/java/bisq/monitor/ThreadGate.java @@ -63,4 +63,12 @@ public synchronized void await() { public void proceed() { lock.countDown(); } + + /** + * Open the gate with no regards on how many locks are still in place. + */ + public void unlock() { + while (lock.getCount() > 0) + lock.countDown(); + } } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 9abf5c9e189..66dc0ff252f 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -43,6 +43,7 @@ import bisq.monitor.OnionParser; import bisq.monitor.Reporter; import bisq.monitor.ThreadGate; +import bisq.network.p2p.CloseConnectionMessage; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; @@ -261,6 +262,11 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { connection.removeMessageListener(this); gate.proceed(); + } else if (networkEnvelope instanceof CloseConnectionMessage) { + gate.unlock(); + } else { + log.warn("Got a message of type <{}>, expected ", + networkEnvelope.getClass().getSimpleName()); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java index 4658349618f..24468c0060c 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java @@ -39,6 +39,7 @@ import bisq.monitor.OnionParser; import bisq.monitor.Reporter; import bisq.monitor.ThreadGate; +import bisq.network.p2p.CloseConnectionMessage; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; @@ -225,6 +226,11 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { connection.removeMessageListener(this); gate.proceed(); + } else if (networkEnvelope instanceof CloseConnectionMessage) { + gate.unlock(); + } else { + log.warn("Got a message of type <{}>, expected ", + networkEnvelope.getClass().getSimpleName()); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 725fd706e7a..0c207f8eceb 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -38,6 +38,7 @@ import bisq.monitor.Reporter; import bisq.monitor.StatisticsHelper; import bisq.monitor.ThreadGate; +import bisq.network.p2p.CloseConnectionMessage; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; @@ -151,6 +152,10 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // open the gate gate.proceed(); + } else if (networkEnvelope instanceof CloseConnectionMessage) { + gate.unlock(); + } else { + log.warn("Got a message of type <{}>, expected ", networkEnvelope.getClass().getSimpleName()); } } From 2f7e74f27f1198bc24422b7ee1aa11d8d962a19a Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Tue, 29 Jan 2019 19:41:41 +0100 Subject: [PATCH 27/40] Reduce log output in productive environment --- monitor/src/main/resources/logback.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 monitor/src/main/resources/logback.xml diff --git a/monitor/src/main/resources/logback.xml b/monitor/src/main/resources/logback.xml new file mode 100644 index 00000000000..65f307ab1fb --- /dev/null +++ b/monitor/src/main/resources/logback.xml @@ -0,0 +1,12 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + \ No newline at end of file From 1cdecd25f85eae3793517b82c7999cf9aab88e36 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 10:50:46 +0100 Subject: [PATCH 28/40] Introduce timeout for ThreadGates --- monitor/src/main/java/bisq/monitor/ThreadGate.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/monitor/src/main/java/bisq/monitor/ThreadGate.java b/monitor/src/main/java/bisq/monitor/ThreadGate.java index 46c0d4e6ade..f7a3383050a 100644 --- a/monitor/src/main/java/bisq/monitor/ThreadGate.java +++ b/monitor/src/main/java/bisq/monitor/ThreadGate.java @@ -18,12 +18,16 @@ package bisq.monitor; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; /** * Gate pattern to help with thread synchronization * * @author Florian Reimair */ +@Slf4j public class ThreadGate { private CountDownLatch lock = new CountDownLatch(0); @@ -52,7 +56,10 @@ public void engage(int numberOfLocks) { public synchronized void await() { while (lock.getCount() > 0) try { - lock.await(); + if (lock.await(lock.getCount(), TimeUnit.MINUTES)) { + log.warn("timeout occured!"); + break; // break the loop + } } catch (InterruptedException ignore) { } } From a363f1545d87d5be7c89ab017e5617f3221835e3 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 10:58:07 +0100 Subject: [PATCH 29/40] Do not flood a node with requests --- .../src/main/java/bisq/monitor/metric/P2PRoundTripTime.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 0c207f8eceb..34f2041c852 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -101,6 +101,9 @@ protected void execute() { samples = new ArrayList<>(); while (samples.size() < Integer.parseInt(configuration.getProperty(SAMPLE_SIZE, "1"))) { + // so we do not get disconnected due to DoS protection mechanisms + Thread.sleep(1000); + nonce = new Random().nextInt(); // close the gate From 2ca5fdc475f9ee4c16b75fe49d778217882ff7e5 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 12:15:46 +0100 Subject: [PATCH 30/40] Fix negation bug --- monitor/src/main/java/bisq/monitor/ThreadGate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitor/src/main/java/bisq/monitor/ThreadGate.java b/monitor/src/main/java/bisq/monitor/ThreadGate.java index f7a3383050a..f384012a760 100644 --- a/monitor/src/main/java/bisq/monitor/ThreadGate.java +++ b/monitor/src/main/java/bisq/monitor/ThreadGate.java @@ -56,7 +56,7 @@ public void engage(int numberOfLocks) { public synchronized void await() { while (lock.getCount() > 0) try { - if (lock.await(lock.getCount(), TimeUnit.MINUTES)) { + if (!lock.await(lock.getCount(), TimeUnit.MINUTES)) { log.warn("timeout occured!"); break; // break the loop } From f64a0baff353ba0f014341ce90ab8d7d6ba35893 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 12:46:50 +0100 Subject: [PATCH 31/40] Open gate if request failed --- monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java | 1 + .../main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java | 1 + monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java | 1 + 3 files changed, 3 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 66dc0ff252f..630024b35ae 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -155,6 +155,7 @@ public void onSuccess(Connection connection) { @Override public void onFailure(@NotNull Throwable throwable) { + gate.proceed(); log.error( "Sending PreliminaryDataRequest failed. That is expected if the peer is offline.\n\tException=" + throwable.getMessage()); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java index 24468c0060c..66e44bb0806 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java @@ -150,6 +150,7 @@ public void onSuccess(Connection connection) { @Override public void onFailure(@NotNull Throwable throwable) { + gate.proceed(); log.error( "Sending PreliminaryDataRequest failed. That is expected if the peer is offline.\n\tException=" + throwable.getMessage()); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 34f2041c852..8d1fb2148e1 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -121,6 +121,7 @@ public void onSuccess(Connection connection) { @Override public void onFailure(@NotNull Throwable throwable) { + gate.proceed(); log.error("Sending ping failed. That is expected if the peer is offline.\n\tException=" + throwable.getMessage()); } From 9f3b3cd1e85003b1432cc5502d82ef9d4501ed78 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 13:44:29 +0100 Subject: [PATCH 32/40] Close unneeded connections --- .../java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java | 3 +++ .../src/main/java/bisq/monitor/metric/P2PRoundTripTime.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java index 66e44bb0806..ed7b57e2817 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java @@ -41,6 +41,7 @@ import bisq.monitor.ThreadGate; import bisq.network.p2p.CloseConnectionMessage; import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.network.CloseConnectionReason; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; import bisq.network.p2p.network.NetworkNode; @@ -226,12 +227,14 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { bucketsPerHost.put(connection.peersNodeAddressProperty().getValue(), buckets); connection.removeMessageListener(this); + connection.shutDown(CloseConnectionReason.APP_SHUT_DOWN); gate.proceed(); } else if (networkEnvelope instanceof CloseConnectionMessage) { gate.unlock(); } else { log.warn("Got a message of type <{}>, expected ", networkEnvelope.getClass().getSimpleName()); + connection.shutDown(CloseConnectionReason.APP_SHUT_DOWN); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index 8d1fb2148e1..f2f0df8fb2f 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -40,6 +40,7 @@ import bisq.monitor.ThreadGate; import bisq.network.p2p.CloseConnectionMessage; import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.network.CloseConnectionReason; import bisq.network.p2p.network.Connection; import bisq.network.p2p.network.MessageListener; import bisq.network.p2p.network.NetworkNode; @@ -154,12 +155,14 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { } connection.removeMessageListener(this); + connection.shutDown(CloseConnectionReason.APP_SHUT_DOWN); // open the gate gate.proceed(); } else if (networkEnvelope instanceof CloseConnectionMessage) { gate.unlock(); } else { log.warn("Got a message of type <{}>, expected ", networkEnvelope.getClass().getSimpleName()); + connection.shutDown(CloseConnectionReason.APP_SHUT_DOWN); } } From d93f3c44cc43c1b9974b8f6d4241b914f18bc053 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 13:46:32 +0100 Subject: [PATCH 33/40] Insert sleep phases to not flood Tor with requests --- .../java/bisq/monitor/metric/P2PRoundTripTime.java | 2 +- .../java/bisq/monitor/reporter/GraphiteReporter.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index f2f0df8fb2f..a8f9d03e45b 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -103,7 +103,7 @@ protected void execute() { while (samples.size() < Integer.parseInt(configuration.getProperty(SAMPLE_SIZE, "1"))) { // so we do not get disconnected due to DoS protection mechanisms - Thread.sleep(1000); + Thread.sleep(200); nonce = new Random().nextInt(); diff --git a/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java b/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java index 890075c75a6..b9f11a1c6bf 100644 --- a/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java +++ b/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java @@ -61,6 +61,16 @@ public void report(Map values, String prefix) { socket.getOutputStream().write(report.getBytes()); socket.close(); + + try { + // give Tor some slack + // TODO maybe use the pickle protocol? + // https://graphite.readthedocs.io/en/latest/feeding-carbon.html + Thread.sleep(100); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); From 834a7dee261a208793375364d1df996fcd1151fa Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 13:48:47 +0100 Subject: [PATCH 34/40] The lockcount does no longer influence the timeout --- monitor/src/main/java/bisq/monitor/ThreadGate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitor/src/main/java/bisq/monitor/ThreadGate.java b/monitor/src/main/java/bisq/monitor/ThreadGate.java index f384012a760..8c9fa27c61f 100644 --- a/monitor/src/main/java/bisq/monitor/ThreadGate.java +++ b/monitor/src/main/java/bisq/monitor/ThreadGate.java @@ -56,7 +56,7 @@ public void engage(int numberOfLocks) { public synchronized void await() { while (lock.getCount() > 0) try { - if (!lock.await(lock.getCount(), TimeUnit.MINUTES)) { + if (!lock.await(90, TimeUnit.SECONDS)) { log.warn("timeout occured!"); break; // break the loop } From 1e8a6e7efc0ab2b36dccbc4fc62040c72396fa85 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 13:49:35 +0100 Subject: [PATCH 35/40] Add torProxyPort setting to example config --- monitor/src/main/resources/metrics.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monitor/src/main/resources/metrics.properties b/monitor/src/main/resources/metrics.properties index bc5c662a30e..76314cda8ff 100644 --- a/monitor/src/main/resources/metrics.properties +++ b/monitor/src/main/resources/metrics.properties @@ -29,16 +29,19 @@ P2PRoundTripTime.enabled=true P2PRoundTripTime.run.interval=100 P2PRoundTripTime.run.sampleSize=5 P2PRoundTripTime.run.hosts=723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 +P2PRoundTripTime.run.torProxyPort=9060 #P2PNetworkLoad Metric P2PNetworkLoad.enabled=true P2PNetworkLoad.run.interval=100 P2PNetworkLoad.run.hosts=723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 +P2PNetworkLoad.run.torProxyPort=9061 #P2PNetworkMessageSnapshot Metric P2PNetworkMessageSnapshot.enabled=true P2PNetworkMessageSnapshot.run.interval=24 P2PNetworkMessageSnapshot.run.hosts=3f3cu2yw7u457ztq.onion:8000, 723ljisnynbtdohi.onion:8000, fl3mmribyxgrv63c.onion:8000 +P2PNetworkMessageSnapshot.run.torProxyPort=9062 #Another Metric From f54ea610d19a3d9865f01b00cd71ae6ef9217939 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Wed, 30 Jan 2019 16:41:00 +0100 Subject: [PATCH 36/40] Remove superfluous code --- .../monitor/metric/P2PNetworkMessageSnapshot.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java index ed7b57e2817..ad2b7557fb3 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java @@ -73,7 +73,6 @@ public class P2PNetworkMessageSnapshot extends Metric implements MessageListener private int nonce; private Map> bucketsPerHost = new ConcurrentHashMap<>(); private Set hashes = new HashSet<>(); - private boolean reportFindings; private final ThreadGate hsReady = new ThreadGate(); private final ThreadGate gate = new ThreadGate(); @@ -119,14 +118,6 @@ protected void execute() { bucketsPerHost.clear(); ArrayList threadList = new ArrayList<>(); - // in case we just started anew, do not report our findings as they contain not - // only the changes since our last run, but a whole lot more dating back even - // till the beginning of bisq. - if (hashes.isEmpty()) - reportFindings = false; - else - reportFindings = true; - // for each configured host for (String current : configuration.getProperty(HOSTS, "").split(",")) { threadList.add(new Thread(new Runnable() { @@ -186,8 +177,7 @@ public void onFailure(@NotNull Throwable throwable) { hashes.clear(); // report our findings iff we have not just started anew - if (reportFindings) - reporter.report(report, "bisq." + getName()); + reporter.report(report, "bisq." + getName()); } @Override From 8cacd92d32fb774bd8b5a5056124c75be59b5660 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 31 Jan 2019 10:50:25 +0100 Subject: [PATCH 37/40] GraphiteReporter can handle non-HS target urls --- .../main/java/bisq/monitor/reporter/GraphiteReporter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java b/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java index b9f11a1c6bf..322ddebb5aa 100644 --- a/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java +++ b/monitor/src/main/java/bisq/monitor/reporter/GraphiteReporter.java @@ -24,7 +24,7 @@ import org.berndpruenster.netlayer.tor.TorSocket; import java.io.IOException; - +import java.net.Socket; import java.util.HashMap; import java.util.Map; @@ -57,7 +57,11 @@ public void report(Map values, String prefix) { try { NodeAddress nodeAddress = OnionParser.getNodeAddress(configuration.getProperty("serviceUrl")); - TorSocket socket = new TorSocket(nodeAddress.getHostName(), nodeAddress.getPort()); + Socket socket; + if (nodeAddress.getFullAddress().contains(".onion")) + socket = new TorSocket(nodeAddress.getHostName(), nodeAddress.getPort()); + else + socket = new Socket(nodeAddress.getHostName(), nodeAddress.getPort()); socket.getOutputStream().write(report.getBytes()); socket.close(); From 3ca0f678f86ea52ec4150336b2f569091dccca24 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Thu, 31 Jan 2019 11:21:19 +0100 Subject: [PATCH 38/40] Monitor release: Showing off --- .../main/java/bisq/monitor/AvailableTor.java | 4 +- .../src/main/java/bisq/monitor/Metric.java | 6 +-- .../src/main/java/bisq/monitor/Monitor.java | 9 ++--- .../bisq/monitor/metric/P2PNetworkLoad.java | 40 +++++-------------- .../metric/P2PNetworkMessageSnapshot.java | 24 ++++------- .../bisq/monitor/metric/P2PRoundTripTime.java | 7 +--- .../bisq/monitor/P2PNetworkLoadTests.java | 11 ++--- .../bisq/monitor/P2PRoundTripTimeTests.java | 11 ++--- 8 files changed, 39 insertions(+), 73 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/AvailableTor.java b/monitor/src/main/java/bisq/monitor/AvailableTor.java index 39041267e13..bd76620c40a 100644 --- a/monitor/src/main/java/bisq/monitor/AvailableTor.java +++ b/monitor/src/main/java/bisq/monitor/AvailableTor.java @@ -18,9 +18,7 @@ package bisq.monitor; import java.io.File; -import java.io.IOException; import org.berndpruenster.netlayer.tor.Tor; -import org.berndpruenster.netlayer.tor.TorCtlException; import bisq.network.p2p.network.TorMode; /** @@ -40,7 +38,7 @@ public AvailableTor(File torWorkingDirectory, String hiddenServiceDirectory) { } @Override - public Tor getTor() throws IOException, TorCtlException { + public Tor getTor() { return Tor.getDefault(); } diff --git a/monitor/src/main/java/bisq/monitor/Metric.java b/monitor/src/main/java/bisq/monitor/Metric.java index 53a2fcd7f37..1417cb29dbc 100644 --- a/monitor/src/main/java/bisq/monitor/Metric.java +++ b/monitor/src/main/java/bisq/monitor/Metric.java @@ -51,7 +51,7 @@ private void disable() { /** * enable execution */ - protected void enable() { + private void enable() { shutdown = false; thread = new Thread(this); @@ -78,7 +78,7 @@ protected Metric(Reporter reporter) { disable(); } - protected boolean enabled() { + boolean enabled() { return !shutdown; } @@ -153,7 +153,7 @@ public void shutdown() { shutdown = true; } - protected void join() throws InterruptedException { + void join() throws InterruptedException { thread.join(); } diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java index ad69bb4aef5..33fb9d06e31 100644 --- a/monitor/src/main/java/bisq/monitor/Monitor.java +++ b/monitor/src/main/java/bisq/monitor/Monitor.java @@ -67,7 +67,7 @@ public static void main(String[] args) throws Throwable { /** * Starts up all configured Metrics. * - * @throws Exception + * @throws Throwable in case something goes wrong */ private void start() throws Throwable { @@ -89,9 +89,7 @@ private void start() throws Throwable { // prepare configuration reload // Note that this is most likely only work on Linux - Signal.handle(new Signal("USR1"), signal -> { - reload(); - }); + Signal.handle(new Signal("USR1"), signal -> reload()); // configure Metrics // - which also starts the metrics if appropriate @@ -141,7 +139,6 @@ private void reload() { for (Metric current : metrics) current.configure(properties); } catch (Exception e) { - // TODO Auto-generated catch block e.printStackTrace(); } } @@ -150,7 +147,7 @@ private void reload() { * Overloads a default set of properties with a file if given * * @return a set of properties - * @throws Exception + * @throws Exception in case something goes wrong */ private Properties getProperties() throws Exception { Properties defaults = new Properties(); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 630024b35ae..a55b7449203 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -27,7 +27,6 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; import org.jetbrains.annotations.NotNull; import com.google.common.util.concurrent.FutureCallback; @@ -77,7 +76,6 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private int nonce; private Map> bucketsPerHost = new ConcurrentHashMap<>(); private Set hashes = new HashSet<>(); - private boolean reportFindings; private final ThreadGate hsReady = new ThreadGate(); private final ThreadGate gate = new ThreadGate(); @@ -87,11 +85,11 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList private class Counter { private int value = 1; - public int value() { + int value() { return value; } - public void increment() { + void increment() { value++; } } @@ -123,20 +121,9 @@ protected void execute() { bucketsPerHost.clear(); ArrayList threadList = new ArrayList<>(); - // in case we just started anew, do not report our findings as they contain not - // only the changes since our last run, but a whole lot more dating back even - // till the beginning of bisq. - if (hashes.isEmpty()) - reportFindings = false; - else - reportFindings = true; - // for each configured host for (String current : configuration.getProperty(HOSTS, "").split(",")) { - threadList.add(new Thread(new Runnable() { - - @Override - public void run() { + threadList.add(new Thread(() -> { try { // parse Url NodeAddress target = OnionParser.getNodeAddress(current); @@ -146,7 +133,7 @@ public void run() { SettableFuture future = networkNode.sendMessage(target, new PreliminaryGetDataRequest(nonce, hashes)); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(Connection connection) { connection.addMessageListener(P2PNetworkLoad.this); @@ -166,8 +153,6 @@ public void onFailure(@NotNull Throwable throwable) { gate.proceed(); // release the gate on error e.printStackTrace(); } - - } }, current)); } @@ -188,7 +173,7 @@ public void onFailure(@NotNull Throwable throwable) { // - assemble diffs Map messagesPerHost = new HashMap<>(); bucketsPerHost.forEach((host, buckets) -> messagesPerHost.put(OnionParser.prettyPrint(host), - buckets.values().stream().collect(Collectors.summingInt(Counter::value)))); + buckets.values().stream().mapToInt(Counter::value).sum())); Optional referenceHost = messagesPerHost.keySet().stream().sorted().findFirst(); Integer referenceValue = messagesPerHost.get(referenceHost.get()); @@ -209,8 +194,10 @@ public void onFailure(@NotNull Throwable throwable) { if (hashes.size() > 150000) hashes.clear(); - // report our findings iff we have not just started anew - if (reportFindings) + // in case we just started anew, do not report our findings as they contain not + // only the changes since our last run, but a whole lot more data dating back even + // to the beginning of bisq. + if (!hashes.isEmpty()) reporter.report(report, "bisq." + getName()); } @@ -222,7 +209,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { GetDataResponse dataResponse = (GetDataResponse) networkEnvelope; Map buckets = new HashMap<>(); final Set dataSet = dataResponse.getDataSet(); - dataSet.stream().forEach(e -> { + dataSet.forEach(e -> { final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); if (protectedStoragePayload == null) { log.warn("StoragePayload was null: {}", networkEnvelope.toString()); @@ -244,7 +231,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { Set persistableNetworkPayloadSet = dataResponse .getPersistableNetworkPayloadSet(); if (persistableNetworkPayloadSet != null) { - persistableNetworkPayloadSet.stream().forEach(persistableNetworkPayload -> { + persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { // memorize message hashes hashes.add(persistableNetworkPayload.getHash()); @@ -273,7 +260,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { @Override public void onTorNodeReady() { - // TODO Auto-generated method stub } @Override @@ -284,13 +270,9 @@ public void onHiddenServicePublished() { @Override public void onSetupFailed(Throwable throwable) { - // TODO Auto-generated method stub - } @Override public void onRequestCustomBridges() { - // TODO Auto-generated method stub - } } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java index ad2b7557fb3..0133c1cc919 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java @@ -55,10 +55,10 @@ import lombok.extern.slf4j.Slf4j; /** - * Contacts a list of hosts and asks them for all the data we do not have. The + * Contacts a list of hosts and asks them for all the data excluding persisted messages. The * answers are then compiled into buckets of message types. Based on these * buckets, the Metric reports (for each host) the message types observed and - * their number along with a relative comparison between all hosts. + * their number. * * @author Florian Reimair * @@ -82,11 +82,11 @@ public class P2PNetworkMessageSnapshot extends Metric implements MessageListener private class Counter { private int value = 1; - public int value() { + int value() { return value; } - public void increment() { + void increment() { value++; } } @@ -120,10 +120,8 @@ protected void execute() { // for each configured host for (String current : configuration.getProperty(HOSTS, "").split(",")) { - threadList.add(new Thread(new Runnable() { + threadList.add(new Thread(() -> { - @Override - public void run() { try { // parse Url NodeAddress target = OnionParser.getNodeAddress(current); @@ -133,7 +131,7 @@ public void run() { SettableFuture future = networkNode.sendMessage(target, new PreliminaryGetDataRequest(nonce, hashes)); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(Connection connection) { connection.addMessageListener(P2PNetworkMessageSnapshot.this); @@ -154,7 +152,6 @@ public void onFailure(@NotNull Throwable throwable) { e.printStackTrace(); } - } }, current)); } @@ -188,7 +185,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { GetDataResponse dataResponse = (GetDataResponse) networkEnvelope; Map buckets = new HashMap<>(); final Set dataSet = dataResponse.getDataSet(); - dataSet.stream().forEach(e -> { + dataSet.forEach(e -> { final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); if (protectedStoragePayload == null) { log.warn("StoragePayload was null: {}", networkEnvelope.toString()); @@ -207,7 +204,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { Set persistableNetworkPayloadSet = dataResponse .getPersistableNetworkPayloadSet(); if (persistableNetworkPayloadSet != null) { - persistableNetworkPayloadSet.stream().forEach(persistableNetworkPayload -> { + persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { // memorize message hashes hashes.add(persistableNetworkPayload.getHash()); @@ -230,7 +227,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { @Override public void onTorNodeReady() { - // TODO Auto-generated method stub } @Override @@ -241,13 +237,9 @@ public void onHiddenServicePublished() { @Override public void onSetupFailed(Throwable throwable) { - // TODO Auto-generated method stub - } @Override public void onRequestCustomBridges() { - // TODO Auto-generated method stub - } } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java index a8f9d03e45b..50915add5b8 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PRoundTripTime.java @@ -113,7 +113,7 @@ protected void execute() { start = System.currentTimeMillis(); SettableFuture future = networkNode.sendMessage(target, new Ping(nonce, 42)); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(Connection connection) { connection.addMessageListener(P2PRoundTripTime.this); @@ -168,7 +168,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { @Override public void onTorNodeReady() { - // TODO Auto-generated method stub } @Override @@ -178,13 +177,9 @@ public void onHiddenServicePublished() { @Override public void onSetupFailed(Throwable throwable) { - // TODO Auto-generated method stub - } @Override public void onRequestCustomBridges() { - // TODO Auto-generated method stub - } } diff --git a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java index 5298402a2ec..db24ea3bc9c 100644 --- a/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PNetworkLoadTests.java @@ -39,7 +39,8 @@ * * @author Florian Reimair */ -public class P2PNetworkLoadTests { +@Disabled +class P2PNetworkLoadTests { /** * A dummy Reporter for development purposes. @@ -53,7 +54,7 @@ public void report(long value) { Assert.fail(); } - public Map hasResults() { + Map hasResults() { return results; } @@ -75,13 +76,13 @@ public void report(Map values, String prefix) { } @BeforeAll - public static void setup() throws TorCtlException { + static void setup() throws TorCtlException { // simulate the tor instance available to all metrics Tor.setDefault(new NativeTor(Monitor.TOR_WORKING_DIR)); } @Test - public void run() throws Exception { + void run() throws Exception { DummyReporter reporter = new DummyReporter(); // configure @@ -109,7 +110,7 @@ public void run() throws Exception { } @AfterAll - public static void cleanup() { + static void cleanup() { Tor tor = Tor.getDefault(); checkNotNull(tor, "tor must not be null"); tor.shutdown(); diff --git a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java index efb682917b5..6c95b7f0cd8 100644 --- a/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java +++ b/monitor/src/test/java/bisq/monitor/P2PRoundTripTimeTests.java @@ -40,7 +40,8 @@ * * @author Florian Reimair */ -public class P2PRoundTripTimeTests { +@Disabled +class P2PRoundTripTimeTests { /** * A dummy Reporter for development purposes. @@ -54,7 +55,7 @@ public void report(long value) { Assert.fail(); } - public Map hasResults() { + Map hasResults() { return results; } @@ -76,14 +77,14 @@ public void report(Map values, String prefix) { } @BeforeAll - public static void setup() throws TorCtlException { + static void setup() throws TorCtlException { // simulate the tor instance available to all metrics Tor.setDefault(new NativeTor(Monitor.TOR_WORKING_DIR)); } @ParameterizedTest @ValueSource(strings = {"default", "3", "4", "10"}) - public void run(String sampleSize) throws Exception { + void run(String sampleSize) throws Exception { DummyReporter reporter = new DummyReporter(); // configure @@ -127,7 +128,7 @@ public void run(String sampleSize) throws Exception { } @AfterAll - public static void cleanup() { + static void cleanup() { Tor tor = Tor.getDefault(); checkNotNull(tor, "tor must not be null"); tor.shutdown(); From f066fc24138ac1e32960cea960c57b3a2fe9d76e Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Fri, 1 Feb 2019 12:56:59 +0100 Subject: [PATCH 39/40] Refactoring: remove unneeded exception --- .../main/java/bisq/monitor/metric/P2PNetworkLoad.java | 9 +++------ .../bisq/monitor/metric/P2PNetworkMessageSnapshot.java | 10 ++++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index a55b7449203..4c13c0a5e3b 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -83,7 +83,7 @@ public class P2PNetworkLoad extends Metric implements MessageListener, SetupList * Efficient way to count message occurrences. */ private class Counter { - private int value = 1; + private int value = 0; int value() { return value; @@ -238,11 +238,8 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // For logging different data types String className = persistableNetworkPayload.getClass().getSimpleName(); - try { - buckets.get(className).increment(); - } catch (NullPointerException nullPointerException) { - buckets.put(className, new Counter()); - } + buckets.putIfAbsent(className, new Counter()); + buckets.get(className).increment(); }); } diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java index 0133c1cc919..cb0a1b31000 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java @@ -80,7 +80,7 @@ public class P2PNetworkMessageSnapshot extends Metric implements MessageListener * Efficient way to count message occurrences. */ private class Counter { - private int value = 1; + private int value = 0; int value() { return value; @@ -194,11 +194,9 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // For logging different data types String className = protectedStoragePayload.getClass().getSimpleName(); - try { - buckets.get(className).increment(); - } catch (NullPointerException nullPointerException) { - buckets.put(className, new Counter()); - } + + buckets.putIfAbsent(className, new Counter()); + buckets.get(className).increment(); }); Set persistableNetworkPayloadSet = dataResponse From d13dceb4ab2550448bc1386d9672c47edb0a3e59 Mon Sep 17 00:00:00 2001 From: Florian Reimair Date: Fri, 1 Feb 2019 12:58:58 +0100 Subject: [PATCH 40/40] Insert null checks --- monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java | 4 ++++ .../java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index 4c13c0a5e3b..4c6a171515b 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -17,6 +17,8 @@ package bisq.monitor.metric; +import static com.google.common.base.Preconditions.checkNotNull; + import java.io.File; import java.net.MalformedURLException; import java.util.ArrayList; @@ -243,6 +245,8 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { }); } + checkNotNull(connection.peersNodeAddressProperty(), + "although the property is nullable, we need it to not be null"); bucketsPerHost.put(connection.peersNodeAddressProperty().getValue(), buckets); connection.removeMessageListener(this); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java index cb0a1b31000..f367a5c9f79 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkMessageSnapshot.java @@ -17,6 +17,8 @@ package bisq.monitor.metric; +import static com.google.common.base.Preconditions.checkNotNull; + import java.io.File; import java.util.ArrayList; import java.util.HashMap; @@ -209,6 +211,8 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { }); } + checkNotNull(connection.peersNodeAddressProperty(), + "although the property is nullable, we need it to not be null"); bucketsPerHost.put(connection.peersNodeAddressProperty().getValue(), buckets); connection.removeMessageListener(this);