From 7319f099d1460686f2552ee59e2676d2ef27d93c Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 20:55:38 -0500 Subject: [PATCH 01/12] Recreate hash from protobuf data To ensure all data are using the new hash method (excluding extraMap) we do not use the hash field from the protobug data but pass null which causes to create the hash new based on the new hash method. --- .../main/java/bisq/core/trade/statistics/TradeStatistics2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java index dc2b3829703..4165844ef9a 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java @@ -214,7 +214,7 @@ public static TradeStatistics2 fromProto(protobuf.TradeStatistics2 proto) { proto.getTradeAmount(), proto.getTradeDate(), proto.getDepositTxId(), - proto.getHash().toByteArray(), + null, // We want to clean up the hashes with the changed hash method in v.1.2.0 so we don't use the value from the proto CollectionUtils.isEmpty(proto.getExtraDataMap()) ? null : proto.getExtraDataMap()); } From a9ed096b8051894a728adb41490d899df7c0716c Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 21:14:03 -0500 Subject: [PATCH 02/12] Add filter.toString method and log filter in case of wrong signature We have atm a invalid filter (prob. some dev polluted a test filter to mainnet) --- .../main/java/bisq/core/filter/Filter.java | 22 +++++++++++++++++++ .../java/bisq/core/filter/FilterManager.java | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/bisq/core/filter/Filter.java b/core/src/main/java/bisq/core/filter/Filter.java index 134dbf52b10..43d366991a1 100644 --- a/core/src/main/java/bisq/core/filter/Filter.java +++ b/core/src/main/java/bisq/core/filter/Filter.java @@ -250,4 +250,26 @@ void setSigAndPubKey(String signatureAsBase64, PublicKey ownerPubKey) { ownerPubKeyBytes = Sig.getPublicKeyBytes(this.ownerPubKey); } + + @Override + public String toString() { + return "Filter{" + + "\n bannedOfferIds=" + bannedOfferIds + + ",\n bannedNodeAddress=" + bannedNodeAddress + + ",\n bannedPaymentAccounts=" + bannedPaymentAccounts + + ",\n bannedCurrencies=" + bannedCurrencies + + ",\n bannedPaymentMethods=" + bannedPaymentMethods + + ",\n arbitrators=" + arbitrators + + ",\n seedNodes=" + seedNodes + + ",\n priceRelayNodes=" + priceRelayNodes + + ",\n preventPublicBtcNetwork=" + preventPublicBtcNetwork + + ",\n btcNodes=" + btcNodes + + ",\n extraDataMap=" + extraDataMap + + ",\n disableDao=" + disableDao + + ",\n disableDaoBelowVersion='" + disableDaoBelowVersion + '\'' + + ",\n disableTradeBelowVersion='" + disableTradeBelowVersion + '\'' + + ",\n mediators=" + mediators + + ",\n refundAgents=" + refundAgents + + "\n}"; + } } diff --git a/core/src/main/java/bisq/core/filter/FilterManager.java b/core/src/main/java/bisq/core/filter/FilterManager.java index a29b43ce13c..5104929f8fc 100644 --- a/core/src/main/java/bisq/core/filter/FilterManager.java +++ b/core/src/main/java/bisq/core/filter/FilterManager.java @@ -297,7 +297,7 @@ private boolean verifySignature(Filter filter) { ECKey.fromPublicOnly(HEX.decode(pubKeyAsHex)).verifyMessage(getHexFromData(filter), filter.getSignatureAsBase64()); return true; } catch (SignatureException e) { - log.warn("verifySignature failed"); + log.warn("verifySignature failed. filter={}", filter); return false; } } From 316dbdf447a095d4d898d2fe195a21b8c42c8651 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 21:16:47 -0500 Subject: [PATCH 03/12] Change log level, add log --- p2p/src/main/java/bisq/network/p2p/network/Connection.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index ef552b9cec3..438bb7136ae 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -358,7 +358,7 @@ public boolean noCapabilityRequiredOrCapabilityIsSupported(Proto msg) { data = ((AddDataMessage) msg).getProtectedStorageEntry().getProtectedStoragePayload(); } // Monitoring nodes have only one capability set, we don't want to log those - log.info("We did not send the message because the peer does not support our required capabilities. " + + log.debug("We did not send the message because the peer does not support our required capabilities. " + "messageClass={}, peer={}, peers supportedCapabilities={}", data.getClass().getSimpleName(), peersNodeAddressOptional, capabilities); } @@ -800,6 +800,8 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { // Capabilities can be empty. We only check for mandatory if we get some capabilities. if (!capabilities.isEmpty() && !Capabilities.hasMandatoryCapability(capabilities)) { + log.warn("We close a connection to an old node. Capabilities of old node: {}", + capabilities.prettyPrint()); shutDown(CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED); return; } From bfd4bebec126d2c70d705cb0772a39e04be99980 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 21:17:37 -0500 Subject: [PATCH 04/12] Refactor: Move code to dump method --- .../statistics/TradeStatisticsManager.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java index 5c423eca6bc..29016f42c9a 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java @@ -76,19 +76,6 @@ public TradeStatisticsManager(P2PService p2PService, } public void onAllServicesInitialized() { - if (dumpStatistics) { - ArrayList fiatCurrencyList = CurrencyUtil.getAllSortedFiatCurrencies().stream() - .map(e -> new CurrencyTuple(e.getCode(), e.getName(), 8)) - .collect(Collectors.toCollection(ArrayList::new)); - jsonFileManager.writeToDisc(Utilities.objectToJson(fiatCurrencyList), "fiat_currency_list"); - - ArrayList cryptoCurrencyList = CurrencyUtil.getAllSortedCryptoCurrencies().stream() - .map(e -> new CurrencyTuple(e.getCode(), e.getName(), 8)) - .collect(Collectors.toCollection(ArrayList::new)); - cryptoCurrencyList.add(0, new CurrencyTuple(Res.getBaseCurrencyCode(), Res.getBaseCurrencyName(), 8)); - jsonFileManager.writeToDisc(Utilities.objectToJson(cryptoCurrencyList), "crypto_currency_list"); - } - p2PService.getP2PDataStorage().addAppendOnlyDataStoreListener(payload -> { if (payload instanceof TradeStatistics2) addToSet((TradeStatistics2) payload); @@ -149,6 +136,17 @@ private void addToSet(TradeStatistics2 tradeStatistics) { private void dump() { if (dumpStatistics) { + ArrayList fiatCurrencyList = CurrencyUtil.getAllSortedFiatCurrencies().stream() + .map(e -> new CurrencyTuple(e.getCode(), e.getName(), 8)) + .collect(Collectors.toCollection(ArrayList::new)); + jsonFileManager.writeToDisc(Utilities.objectToJson(fiatCurrencyList), "fiat_currency_list"); + + ArrayList cryptoCurrencyList = CurrencyUtil.getAllSortedCryptoCurrencies().stream() + .map(e -> new CurrencyTuple(e.getCode(), e.getName(), 8)) + .collect(Collectors.toCollection(ArrayList::new)); + cryptoCurrencyList.add(0, new CurrencyTuple(Res.getBaseCurrencyCode(), Res.getBaseCurrencyName(), 8)); + jsonFileManager.writeToDisc(Utilities.objectToJson(cryptoCurrencyList), "crypto_currency_list"); + // We store the statistics as json so it is easy for further processing (e.g. for web based services) // TODO This is just a quick solution for storing to one file. // 1 statistic entry has 500 bytes as json. From 373e632b90706284dee8d3348038acd9495a1301 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 22:01:01 -0500 Subject: [PATCH 05/12] Add TRADE_STATISTICS_HASH_UPDATE capability We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data. --- common/src/main/java/bisq/common/app/Capability.java | 4 ++-- .../java/bisq/core/trade/statistics/TradeStatistics2.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/bisq/common/app/Capability.java b/common/src/main/java/bisq/common/app/Capability.java index 251240ea3d1..33040bbfa40 100644 --- a/common/src/main/java/bisq/common/app/Capability.java +++ b/common/src/main/java/bisq/common/app/Capability.java @@ -39,6 +39,6 @@ public enum Capability { SIGNED_ACCOUNT_AGE_WITNESS, // Supports the signed account age witness feature MEDIATION, // Supports mediation feature - REFUND_AGENT // Supports refund agents - ; + REFUND_AGENT, // Supports refund agents + TRADE_STATISTICS_HASH_UPDATE // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data. } diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java index 4165844ef9a..12775b719a2 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java @@ -235,12 +235,13 @@ public boolean verifyHashSize() { } // With v1.2.0 we changed the way how the hash is created. To not create too heavy load for seed nodes from - // requests from old nodes we use the SIGNED_ACCOUNT_AGE_WITNESS capability to send trade statistics only to new + // requests from old nodes we use the TRADE_STATISTICS_HASH_UPDATE capability to send trade statistics only to new // nodes. As trade statistics are only used for informational purpose it will not have any critical issue for the - // old nodes beside that they don't see the latest trades. + // old nodes beside that they don't see the latest trades. We added TRADE_STATISTICS_HASH_UPDATE in v1.2.2 to fix a + // problem of not handling the hashes correctly. @Override public Capabilities getRequiredCapabilities() { - return new Capabilities(Capability.SIGNED_ACCOUNT_AGE_WITNESS); + return new Capabilities(Capability.TRADE_STATISTICS_HASH_UPDATE); } @@ -282,7 +283,6 @@ public boolean isValid() { return tradeAmount > 0 && tradePrice > 0 && !excludedFailedTrade; } - @Override public String toString() { return "TradeStatistics2{" + From 2ddfcc8e87c38bd17d98a6841f3503162f6c4e74 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 22:05:21 -0500 Subject: [PATCH 06/12] Add logs for size of data exchange messages --- .../p2p/peers/getdata/messages/GetDataResponse.java | 9 +++++++-- .../peers/getdata/messages/GetUpdatedDataRequest.java | 11 ++++++++++- .../getdata/messages/PreliminaryGetDataRequest.java | 9 ++++++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java index 03e1a09ff23..726952429d9 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java @@ -108,12 +108,17 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { .map(PersistableNetworkPayload::toProtoMessage) .collect(Collectors.toList()))); - return getNetworkEnvelopeBuilder() + protobuf.NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setGetDataResponse(builder) .build(); + log.info("Sending a GetDataResponse with size = {}", proto.toByteArray().length); + return proto; } - public static GetDataResponse fromProto(protobuf.GetDataResponse proto, NetworkProtoResolver resolver, int messageVersion) { + public static GetDataResponse fromProto(protobuf.GetDataResponse proto, + NetworkProtoResolver resolver, + int messageVersion) { + log.info("Received a GetDataResponse with size = {}", proto.toByteArray().length); Set dataSet = new HashSet<>( proto.getDataSetList().stream() .map(entry -> (ProtectedStorageEntry) resolver.fromProto(entry)) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java index 166c3cd0570..56975b59d47 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java @@ -30,9 +30,15 @@ import lombok.EqualsAndHashCode; import lombok.Value; +import lombok.extern.slf4j.Slf4j; import static com.google.common.base.Preconditions.checkNotNull; + + +import protobuf.NetworkEnvelope; + +@Slf4j @EqualsAndHashCode(callSuper = true) @Value public final class GetUpdatedDataRequest extends GetDataRequest implements SendersNodeAddressMessage { @@ -72,12 +78,15 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { .map(ByteString::copyFrom) .collect(Collectors.toList())); - return getNetworkEnvelopeBuilder() + NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setGetUpdatedDataRequest(builder) .build(); + log.info("Sending a GetUpdatedDataRequest with size = {}", proto.toByteArray().length); + return proto; } public static GetUpdatedDataRequest fromProto(protobuf.GetUpdatedDataRequest proto, int messageVersion) { + log.info("Received a GetUpdatedDataRequest with size = {}", proto.toByteArray().length); return new GetUpdatedDataRequest(NodeAddress.fromProto(proto.getSenderNodeAddress()), proto.getNonce(), ProtoUtil.byteSetFromProtoByteStringList(proto.getExcludedKeysList()), diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java index 07d5f87464c..cd7d83eb3c9 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java @@ -36,6 +36,10 @@ import javax.annotation.Nullable; + + +import protobuf.NetworkEnvelope; + @Slf4j @EqualsAndHashCode(callSuper = true) @Value @@ -72,12 +76,15 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { Optional.ofNullable(supportedCapabilities).ifPresent(e -> builder.addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities))); - return getNetworkEnvelopeBuilder() + NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setPreliminaryGetDataRequest(builder) .build(); + log.info("Sending a PreliminaryGetDataRequest with size = {}", proto.toByteArray().length); + return proto; } public static PreliminaryGetDataRequest fromProto(protobuf.PreliminaryGetDataRequest proto, int messageVersion) { + log.info("Received a PreliminaryGetDataRequest with size = {}", proto.toByteArray().length); Capabilities supportedCapabilities = proto.getSupportedCapabilitiesList().isEmpty() ? null : Capabilities.fromIntList(proto.getSupportedCapabilitiesList()); From 1d7712606545b5d3aad7867d0a30e6af187696f3 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 22:09:08 -0500 Subject: [PATCH 07/12] Add more data in log --- .../main/java/bisq/network/p2p/network/Connection.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index 438bb7136ae..86ddafc06dc 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -800,8 +800,12 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { // Capabilities can be empty. We only check for mandatory if we get some capabilities. if (!capabilities.isEmpty() && !Capabilities.hasMandatoryCapability(capabilities)) { - log.warn("We close a connection to an old node. Capabilities of old node: {}", - capabilities.prettyPrint()); + String senderNodeAddress = networkEnvelope instanceof SendersNodeAddressMessage ? + ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress().getFullAddress() : + "[unknown address]"; + log.warn("We close a connection to old node {}. " + + "Capabilities of old node: {}, networkEnvelope class name={}", + senderNodeAddress, capabilities.prettyPrint(), networkEnvelope.getClass().getSimpleName()); shutDown(CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED); return; } From 559c7a2dc6b709c001661687ad651a2540796313 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 22:39:36 -0500 Subject: [PATCH 08/12] Improve logs --- p2p/src/main/java/bisq/network/p2p/network/Connection.java | 2 +- .../network/p2p/peers/getdata/messages/GetDataResponse.java | 4 ++-- .../p2p/peers/getdata/messages/GetUpdatedDataRequest.java | 4 ++-- .../p2p/peers/getdata/messages/PreliminaryGetDataRequest.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index 86ddafc06dc..d25bdeb5f10 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -803,7 +803,7 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { String senderNodeAddress = networkEnvelope instanceof SendersNodeAddressMessage ? ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress().getFullAddress() : "[unknown address]"; - log.warn("We close a connection to old node {}. " + + log.info("We close a connection to old node {}. " + "Capabilities of old node: {}, networkEnvelope class name={}", senderNodeAddress, capabilities.prettyPrint(), networkEnvelope.getClass().getSimpleName()); shutDown(CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java index 726952429d9..fce89a42d10 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java @@ -111,14 +111,14 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { protobuf.NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setGetDataResponse(builder) .build(); - log.info("Sending a GetDataResponse with size = {}", proto.toByteArray().length); + log.info("Sending a GetDataResponse with size = {} bytes", proto.toByteArray().length); return proto; } public static GetDataResponse fromProto(protobuf.GetDataResponse proto, NetworkProtoResolver resolver, int messageVersion) { - log.info("Received a GetDataResponse with size = {}", proto.toByteArray().length); + log.info("Received a GetDataResponse with size = {} bytes", proto.toByteArray().length); Set dataSet = new HashSet<>( proto.getDataSetList().stream() .map(entry -> (ProtectedStorageEntry) resolver.fromProto(entry)) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java index 56975b59d47..0b5031e5495 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java @@ -81,12 +81,12 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setGetUpdatedDataRequest(builder) .build(); - log.info("Sending a GetUpdatedDataRequest with size = {}", proto.toByteArray().length); + log.info("Sending a GetUpdatedDataRequest with size = {} bytes", proto.toByteArray().length); return proto; } public static GetUpdatedDataRequest fromProto(protobuf.GetUpdatedDataRequest proto, int messageVersion) { - log.info("Received a GetUpdatedDataRequest with size = {}", proto.toByteArray().length); + log.info("Received a GetUpdatedDataRequest with size = {} bytes", proto.toByteArray().length); return new GetUpdatedDataRequest(NodeAddress.fromProto(proto.getSenderNodeAddress()), proto.getNonce(), ProtoUtil.byteSetFromProtoByteStringList(proto.getExcludedKeysList()), diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java index cd7d83eb3c9..ac8b4c39342 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java @@ -79,12 +79,12 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setPreliminaryGetDataRequest(builder) .build(); - log.info("Sending a PreliminaryGetDataRequest with size = {}", proto.toByteArray().length); + log.info("Sending a PreliminaryGetDataRequest with size = {} bytes", proto.toByteArray().length); return proto; } public static PreliminaryGetDataRequest fromProto(protobuf.PreliminaryGetDataRequest proto, int messageVersion) { - log.info("Received a PreliminaryGetDataRequest with size = {}", proto.toByteArray().length); + log.info("Received a PreliminaryGetDataRequest with size = {} bytes", proto.toByteArray().length); Capabilities supportedCapabilities = proto.getSupportedCapabilitiesList().isEmpty() ? null : Capabilities.fromIntList(proto.getSupportedCapabilitiesList()); From cfe218ae70931473aa74a238f5677809de0eb79f Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 23:08:16 -0500 Subject: [PATCH 09/12] Fix wrong msg in log, cahnge log level --- p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index add6e486789..0b918328ba9 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -353,8 +353,7 @@ private boolean checkMaxConnections() { connection.shutDown(CloseConnectionReason.TOO_MANY_CONNECTIONS_OPEN, () -> UserThread.runAfter(this::checkMaxConnections, 100, TimeUnit.MILLISECONDS)); return true; } else { - log.warn("No candidates found to remove (That case should not be possible as we use in the " + - "last case all connections).\n\t" + + log.debug("No candidates found to remove.\n\t" + "size={}, allConnections={}", size, allConnections); return false; } From d8776f3323f041d8892a90de6ae4f2bbd655a6a7 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 23:12:20 -0500 Subject: [PATCH 10/12] Add check for depositTxId not empty --- .../main/java/bisq/core/trade/statistics/TradeStatistics2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java index 12775b719a2..feb0b6ed14d 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java @@ -280,7 +280,7 @@ public boolean isValid() { // Since the trade wasn't executed it's better to filter it out to avoid it having an undue influence on the // BSQ trade stats. boolean excludedFailedTrade = offerId.equals("6E5KOI6O-3a06a037-6f03-4bfa-98c2-59f49f73466a-112"); - return tradeAmount > 0 && tradePrice > 0 && !excludedFailedTrade; + return tradeAmount > 0 && tradePrice > 0 && !excludedFailedTrade && !depositTxId.isEmpty(); } @Override From 8c843d1abbf7441a54597207bd18ffdab6dac3d5 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 23:13:25 -0500 Subject: [PATCH 11/12] Remove check for duplicates As we recreate the hash for all trade stat objects we don't need that check anymore. --- .../TradeStatistics2StorageService.java | 15 --------- .../statistics/TradeStatisticsManager.java | 33 +++---------------- 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java index a06ca30c67b..60673e07320 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java @@ -29,8 +29,6 @@ import java.io.File; -import java.util.Collection; -import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -70,19 +68,6 @@ public boolean canHandle(PersistableNetworkPayload payload) { return payload instanceof TradeStatistics2; } - Collection cleanupMap(Collection collection) { - Map tempMap = new HashMap<>(); - // We recreate the hash as there have been duplicates from diff. extraMap entries introduced at software updates - collection.forEach(item -> tempMap.putIfAbsent(new P2PDataStorage.ByteArray(item.createHash()), item)); - - Map map = getMap(); - map.clear(); - map.putAll(tempMap); - persist(); - - return tempMap.values(); - } - /////////////////////////////////////////////////////////////////////////////////////////// // Protected diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java index 29016f42c9a..e66c91b3e9b 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java @@ -39,11 +39,8 @@ import java.io.File; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -57,7 +54,6 @@ public class TradeStatisticsManager { private final TradeStatistics2StorageService tradeStatistics2StorageService; private final boolean dumpStatistics; private final ObservableSet observableTradeStatisticsSet = FXCollections.observableSet(); - private int duplicates = 0; @Inject public TradeStatisticsManager(P2PService p2PService, @@ -81,33 +77,12 @@ public void onAllServicesInitialized() { addToSet((TradeStatistics2) payload); }); - Map map = new HashMap<>(); - AtomicInteger origSize = new AtomicInteger(); - p2PService.getP2PDataStorage().getAppendOnlyDataStoreMap().values().stream() + Set collect = tradeStatistics2StorageService.getMap().values().stream() .filter(e -> e instanceof TradeStatistics2) .map(e -> (TradeStatistics2) e) .filter(TradeStatistics2::isValid) - .forEach(tradeStatistics -> { - origSize.getAndIncrement(); - TradeStatistics2 prevValue = map.putIfAbsent(tradeStatistics.getOfferId(), tradeStatistics); - if (prevValue != null) { - duplicates++; - } - }); - - Collection items = map.values(); - // At startup we check if we have duplicate entries. This might be the case from software updates when we - // introduced new entries to the extraMap. As that map is for flexibility in updates we keep it excluded from - // json so that it will not cause duplicates anymore. Until all users have updated we keep the cleanup code. - // Should not be needed later anymore, but will also not hurt if no duplicates exist. - if (duplicates > 0) { - long ts = System.currentTimeMillis(); - items = tradeStatistics2StorageService.cleanupMap(items); - log.info("We found {} duplicate entries. Size of map entries before and after cleanup: {} / {}. Cleanup took {} ms.", - duplicates, origSize, items.size(), System.currentTimeMillis() - ts); - } - - observableTradeStatisticsSet.addAll(items); + .collect(Collectors.toSet()); + observableTradeStatisticsSet.addAll(collect); priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); From 407a32864f5e8ef068a8d254663f34770ea5a5fe Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 30 Oct 2019 23:14:09 -0500 Subject: [PATCH 12/12] Add logs --- .../p2p/peers/getdata/GetDataRequestHandler.java | 10 +++++++--- .../network/p2p/peers/getdata/RequestDataHandler.java | 10 ++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java index e46fd4086e9..d6e2259b510 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java @@ -90,6 +90,7 @@ public GetDataRequestHandler(NetworkNode networkNode, P2PDataStorage dataStorage /////////////////////////////////////////////////////////////////////////////////////////// public void handle(GetDataRequest getDataRequest, final Connection connection) { + long ts = System.currentTimeMillis(); GetDataResponse getDataResponse = new GetDataResponse(getFilteredProtectedStorageEntries(getDataRequest, connection), getFilteredPersistableNetworkPayload(getDataRequest, connection), getDataRequest.getNonce(), @@ -105,7 +106,7 @@ public void handle(GetDataRequest getDataRequest, final Connection connection) { } SettableFuture future = networkNode.sendMessage(connection, getDataResponse); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(Connection connection) { if (!stopped) { @@ -130,9 +131,11 @@ public void onFailure(@NotNull Throwable throwable) { } } }); + log.info("handle GetDataRequest took {} ms", System.currentTimeMillis() - ts); } - private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, Connection connection) { + private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, + Connection connection) { final Set tempLookupSet = new HashSet<>(); Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); @@ -144,7 +147,8 @@ private Set getFilteredPersistableNetworkPayload(GetD .collect(Collectors.toSet()); } - private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, Connection connection) { + private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, + Connection connection) { final Set filteredDataSet = new HashSet<>(); final Set lookupSet = new HashSet<>(); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java index 3f10d6511d6..f0c972780e3 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java @@ -203,6 +203,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof GetDataResponse) { if (connection.getPeersNodeAddressOptional().isPresent() && connection.getPeersNodeAddressOptional().get().equals(peersNodeAddress)) { if (!stopped) { + long ts1 = System.currentTimeMillis(); GetDataResponse getDataResponse = (GetDataResponse) networkEnvelope; final Set dataSet = getDataResponse.getDataSet(); Set persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet(); @@ -219,7 +220,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { final NodeAddress sender = connection.getPeersNodeAddressOptional().get(); - long ts = System.currentTimeMillis(); + long ts2 = System.currentTimeMillis(); AtomicInteger counter = new AtomicInteger(); dataSet.forEach(e -> { // We don't broadcast here (last param) as we are only connected to the seed node and would be pointless @@ -227,14 +228,14 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { counter.getAndIncrement(); }); - log.info("Processing {} protectedStorageEntries took {} ms.", counter.get(), System.currentTimeMillis() - ts); + log.info("Processing {} protectedStorageEntries took {} ms.", counter.get(), System.currentTimeMillis() - ts2); /* // engage the firstRequest logic only if we are a seed node. Normal clients get here twice at most. if (!Capabilities.app.containsAll(Capability.SEED_NODE)) firstRequest = true;*/ if (persistableNetworkPayloadSet != null /*&& firstRequest*/) { - ts = System.currentTimeMillis(); + ts2 = System.currentTimeMillis(); persistableNetworkPayloadSet.forEach(e -> { if (e instanceof LazyProcessedPayload) { // We use an optimized method as many checks are not required in that case to avoid @@ -259,7 +260,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { initialRequestApplied = true; log.info("Processing {} persistableNetworkPayloads took {} ms.", - persistableNetworkPayloadSet.size(), System.currentTimeMillis() - ts); + persistableNetworkPayloadSet.size(), System.currentTimeMillis() - ts2); } cleanup(); @@ -272,6 +273,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { "We drop that message. nonce={} / requestNonce={}", nonce, getDataResponse.getRequestNonce()); } + log.info("Processing GetDataResponse took {} ms", System.currentTimeMillis() - ts1); } else { log.warn("We have stopped already. We ignore that onDataRequest call."); }