diff --git a/core/src/main/java/bisq/core/account/sign/SignedWitness.java b/core/src/main/java/bisq/core/account/sign/SignedWitness.java index e5af940470c..d2268400ac6 100644 --- a/core/src/main/java/bisq/core/account/sign/SignedWitness.java +++ b/core/src/main/java/bisq/core/account/sign/SignedWitness.java @@ -20,7 +20,7 @@ import bisq.network.p2p.storage.P2PDataStorage; import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; import bisq.network.p2p.storage.payload.DateTolerantPayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.common.app.Capabilities; @@ -45,7 +45,7 @@ // Supports signatures made from EC key (arbitrators) and signature created with DSA key. @Slf4j @Value -public class SignedWitness implements LazyProcessedPayload, PersistableNetworkPayload, PersistableEnvelope, +public class SignedWitness implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, PersistableEnvelope, DateTolerantPayload, CapabilityRequiringPayload { public enum VerificationMethod { diff --git a/core/src/main/java/bisq/core/account/witness/AccountAgeWitness.java b/core/src/main/java/bisq/core/account/witness/AccountAgeWitness.java index 9578df22973..07f05096602 100644 --- a/core/src/main/java/bisq/core/account/witness/AccountAgeWitness.java +++ b/core/src/main/java/bisq/core/account/witness/AccountAgeWitness.java @@ -19,7 +19,7 @@ import bisq.network.p2p.storage.P2PDataStorage; import bisq.network.p2p.storage.payload.DateTolerantPayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.common.proto.persistable.PersistableEnvelope; @@ -40,7 +40,7 @@ // so only the newly added objects since the last release will be retrieved over the P2P network. @Slf4j @Value -public class AccountAgeWitness implements LazyProcessedPayload, PersistableNetworkPayload, PersistableEnvelope, DateTolerantPayload { +public class AccountAgeWitness implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, PersistableEnvelope, DateTolerantPayload { private static final long TOLERANCE = TimeUnit.DAYS.toMillis(1); private final byte[] hash; // Ripemd160(Sha256(concatenated accountHash, signature and sigPubKey)); 20 bytes diff --git a/core/src/main/java/bisq/core/dao/governance/proposal/storage/temp/TempProposalPayload.java b/core/src/main/java/bisq/core/dao/governance/proposal/storage/temp/TempProposalPayload.java index 6ea17a819d1..f5334fa6030 100644 --- a/core/src/main/java/bisq/core/dao/governance/proposal/storage/temp/TempProposalPayload.java +++ b/core/src/main/java/bisq/core/dao/governance/proposal/storage/temp/TempProposalPayload.java @@ -20,7 +20,7 @@ import bisq.core.dao.state.model.governance.Proposal; import bisq.network.p2p.storage.payload.ExpirablePayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.crypto.Sig; @@ -55,7 +55,7 @@ @Getter @EqualsAndHashCode @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class TempProposalPayload implements LazyProcessedPayload, ProtectedStoragePayload, +public class TempProposalPayload implements ProcessOncePersistableNetworkPayload, ProtectedStoragePayload, ExpirablePayload, PersistablePayload { protected final Proposal proposal; diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java index 54412a5de10..41c77aeb36e 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java @@ -24,7 +24,7 @@ import bisq.core.offer.OfferPayload; import bisq.network.p2p.storage.payload.ExpirablePayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.crypto.Sig; @@ -60,7 +60,7 @@ @Slf4j @EqualsAndHashCode(exclude = {"signaturePubKeyBytes"}) @Value -public final class TradeStatistics implements LazyProcessedPayload, ProtectedStoragePayload, ExpirablePayload, PersistablePayload { +public final class TradeStatistics implements ProcessOncePersistableNetworkPayload, ProtectedStoragePayload, ExpirablePayload, PersistablePayload { private final OfferPayload.Direction direction; private final String baseCurrency; private final String counterCurrency; diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java index 601284f4e74..ba1f6abaf25 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java @@ -25,7 +25,7 @@ import bisq.core.offer.OfferUtil; import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.common.app.Capabilities; @@ -63,7 +63,7 @@ @Slf4j @Value -public final class TradeStatistics2 implements LazyProcessedPayload, PersistableNetworkPayload, PersistableEnvelope, CapabilityRequiringPayload { +public final class TradeStatistics2 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, PersistableEnvelope, CapabilityRequiringPayload { //We don't support arbitrators anymore so this entry will be only for pre v1.2. trades @Deprecated diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java index edebbf13d44..21a17fd9945 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java @@ -221,20 +221,15 @@ protected boolean treatMessage(NetworkEnvelope networkEnvelope, Connection conne versions.log(protectedStoragePayload); }); - Set persistableNetworkPayloadSet = dataResponse - .getPersistableNetworkPayloadSet(); - if (persistableNetworkPayloadSet != null) { - persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { + dataResponse.getPersistableNetworkPayloadSet().forEach(persistableNetworkPayload -> { + // memorize message hashes + //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; + //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); - // memorize message hashes - //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; - //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); + //hashes.add(bytes); - //hashes.add(bytes); - - hashes.add(persistableNetworkPayload.getHash()); - }); - } + hashes.add(persistableNetworkPayload.getHash()); + }); bucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), result); versionBucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), versions); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java index 77462aa4e5e..aefb227189c 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java @@ -333,20 +333,15 @@ protected boolean treatMessage(NetworkEnvelope networkEnvelope, Connection conne result.log(protectedStoragePayload); }); - Set persistableNetworkPayloadSet = dataResponse - .getPersistableNetworkPayloadSet(); - if (persistableNetworkPayloadSet != null) { - persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { + dataResponse.getPersistableNetworkPayloadSet().forEach(persistableNetworkPayload -> { + // memorize message hashes + //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; + //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); - // memorize message hashes - //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; - //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); + //hashes.add(bytes); - //hashes.add(bytes); - - hashes.add(persistableNetworkPayload.getHash()); - }); - } + hashes.add(persistableNetworkPayload.getHash()); + }); bucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), result); return true; diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java index 0972cf32c8c..bb2c4949f9f 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java @@ -22,27 +22,17 @@ import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.peers.getdata.messages.GetDataRequest; import bisq.network.p2p.peers.getdata.messages.GetDataResponse; -import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; import bisq.network.p2p.storage.P2PDataStorage; -import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; -import bisq.network.p2p.storage.payload.PersistableNetworkPayload; -import bisq.network.p2p.storage.payload.ProtectedStorageEntry; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.Timer; import bisq.common.UserThread; -import bisq.common.util.Utilities; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; @@ -51,8 +41,8 @@ @Slf4j public class GetDataRequestHandler { private static final long TIMEOUT = 90; - private static final int MAX_ENTRIES = 10000; + private static final int MAX_ENTRIES = 10000; /////////////////////////////////////////////////////////////////////////////////////////// // Listener @@ -93,10 +83,35 @@ public GetDataRequestHandler(NetworkNode networkNode, P2PDataStorage dataStorage public void handle(GetDataRequest getDataRequest, final Connection connection) { long ts = System.currentTimeMillis(); - GetDataResponse getDataResponse = new GetDataResponse(getFilteredProtectedStorageEntries(getDataRequest, connection), - getFilteredPersistableNetworkPayload(getDataRequest, connection), - getDataRequest.getNonce(), - getDataRequest instanceof GetUpdatedDataRequest); + String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() + .map(e -> "node address " + e.getFullAddress()) + .orElseGet(() -> "connection UID " + connection.getUid()); + + AtomicBoolean outPersistableNetworkPayloadOutputTruncated = new AtomicBoolean(false); + AtomicBoolean outProtectedStoragePayloadOutputTruncated = new AtomicBoolean(false); + GetDataResponse getDataResponse = dataStorage.buildGetDataResponse( + getDataRequest, + MAX_ENTRIES, + outPersistableNetworkPayloadOutputTruncated, + outProtectedStoragePayloadOutputTruncated, + connection.getCapabilities()); + + if (outPersistableNetworkPayloadOutputTruncated.get()) { + log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " + + "entries to get delivered. We limited the entries for the response to {} entries", + connectionInfo, MAX_ENTRIES); + } + + if (outProtectedStoragePayloadOutputTruncated.get()) { + log.warn("The getData request from peer with {} caused too much ProtectedStorageEntry " + + "entries to get delivered. We limited the entries for the response to {} entries", + connectionInfo, MAX_ENTRIES); + } + + log.info("The getDataResponse to peer with {} contains {} ProtectedStorageEntries and {} PersistableNetworkPayloads", + connectionInfo, + getDataResponse.getDataSet().size(), + getDataResponse.getPersistableNetworkPayloadSet().size()); if (timeoutTimer == null) { timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions @@ -136,81 +151,6 @@ public void onFailure(@NotNull Throwable throwable) { log.info("handle GetDataRequest took {} ms", System.currentTimeMillis() - ts); } - private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, - Connection connection) { - Set tempLookupSet = new HashSet<>(); - String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() - .map(e -> "node address " + e.getFullAddress()) - .orElseGet(() -> "connection UID " + connection.getUid()); - - Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); - Set result = dataStorage.getAppendOnlyDataStoreMap().entrySet().stream() - .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) - .filter(e -> maxSize.decrementAndGet() >= 0) - .map(Map.Entry::getValue) - .filter(connection::noCapabilityRequiredOrCapabilityIsSupported) - .filter(payload -> { - boolean notContained = tempLookupSet.add(new P2PDataStorage.ByteArray(payload.getHash())); - return notContained; - }) - .collect(Collectors.toSet()); - if (maxSize.get() <= 0) { - log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " + - "entries to get delivered. We limited the entries for the response to {} entries", - connectionInfo, MAX_ENTRIES); - } - log.info("The getData request from peer with {} contains {} PersistableNetworkPayload entries ", - connectionInfo, result.size()); - return result; - } - - private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, - Connection connection) { - Set filteredDataSet = new HashSet<>(); - Set lookupSet = new HashSet<>(); - String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() - .map(e -> "node address " + e.getFullAddress()) - .orElseGet(() -> "connection UID " + connection.getUid()); - - AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); - Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - Set filteredSet = dataStorage.getMap().entrySet().stream() - .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) - .filter(e -> maxSize.decrementAndGet() >= 0) - .map(Map.Entry::getValue) - .collect(Collectors.toSet()); - if (maxSize.get() <= 0) { - log.warn("The getData request from peer with {} caused too much ProtectedStorageEntry " + - "entries to get delivered. We limited the entries for the response to {} entries", - connectionInfo, MAX_ENTRIES); - } - log.info("getFilteredProtectedStorageEntries " + filteredSet.size()); - - for (ProtectedStorageEntry protectedStorageEntry : filteredSet) { - final ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); - boolean doAdd = false; - if (protectedStoragePayload instanceof CapabilityRequiringPayload) { - if (connection.getCapabilities().containsAll(((CapabilityRequiringPayload) protectedStoragePayload).getRequiredCapabilities())) - doAdd = true; - else - log.debug("We do not send the message to the peer because they do not support the required capability for that message type.\n" + - "storagePayload is: " + Utilities.toTruncatedString(protectedStoragePayload)); - } else { - doAdd = true; - } - if (doAdd) { - boolean notContained = lookupSet.add(protectedStoragePayload.hashCode()); - if (notContained) - filteredDataSet.add(protectedStorageEntry); - } - } - - log.info("The getData request from peer with {} contains {} ProtectedStorageEntry entries ", - connectionInfo, filteredDataSet.size()); - return filteredDataSet; - } - public void stop() { cleanup(); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java index a116d171b7d..4d1ada32941 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java @@ -25,10 +25,7 @@ import bisq.network.p2p.peers.PeerManager; import bisq.network.p2p.peers.getdata.messages.GetDataRequest; import bisq.network.p2p.peers.getdata.messages.GetDataResponse; -import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; -import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; import bisq.network.p2p.storage.P2PDataStorage; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStorageEntry; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; @@ -47,8 +44,6 @@ import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -58,7 +53,6 @@ @Slf4j class RequestDataHandler implements MessageListener { private static final long TIMEOUT = 90; - private static boolean initialRequestApplied = false; private NodeAddress peersNodeAddress; /* @@ -124,25 +118,10 @@ void requestData(NodeAddress nodeAddress, boolean isPreliminaryDataRequest) { if (!stopped) { GetDataRequest getDataRequest; - // We collect the keys of the PersistableNetworkPayload items so we exclude them in our request. - // PersistedStoragePayload items don't get removed, so we don't have an issue with the case that - // an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would - // miss that event if we do not load the full set or use some delta handling. - Set excludedKeys = dataStorage.getAppendOnlyDataStoreMap().keySet().stream() - .map(e -> e.bytes) - .collect(Collectors.toSet()); - - Set excludedKeysFromPersistedEntryMap = dataStorage.getMap().keySet() - .stream() - .map(e -> e.bytes) - .collect(Collectors.toSet()); - - excludedKeys.addAll(excludedKeysFromPersistedEntryMap); - if (isPreliminaryDataRequest) - getDataRequest = new PreliminaryGetDataRequest(nonce, excludedKeys); + getDataRequest = dataStorage.buildPreliminaryGetDataRequest(nonce); else - getDataRequest = new GetUpdatedDataRequest(networkNode.getNodeAddress(), nonce, excludedKeys); + getDataRequest = dataStorage.buildGetUpdatedDataRequest(networkNode.getNodeAddress(), nonce); if (timeoutTimer == null) { timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions @@ -207,7 +186,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { GetDataResponse getDataResponse = (GetDataResponse) networkEnvelope; final Set dataSet = getDataResponse.getDataSet(); Set persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet(); - logContents(networkEnvelope, dataSet, persistableNetworkPayloadSet); if (getDataResponse.getRequestNonce() == nonce) { @@ -218,50 +196,8 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { return; } - final NodeAddress sender = connection.getPeersNodeAddressOptional().get(); - - long ts2 = System.currentTimeMillis(); - AtomicInteger counter = new AtomicInteger(); - dataSet.forEach(e -> { - // We don't broadcast here (last param) as we are only connected to the seed node and would be pointless - dataStorage.addProtectedStorageEntry(e, sender, null, false, false); - counter.getAndIncrement(); - - }); - log.info("Processing {} protectedStorageEntries took {} ms.", counter.get(), System.currentTimeMillis() - ts2); - - /* // engage the firstRequest logic only if we are a seed node. Normal clients get here twice at most. - if (!Capabilities.app.containsAll(Capability.SEED_NODE)) - firstRequest = true;*/ - - if (persistableNetworkPayloadSet != null /*&& firstRequest*/) { - ts2 = System.currentTimeMillis(); - persistableNetworkPayloadSet.forEach(e -> { - if (e instanceof LazyProcessedPayload) { - // We use an optimized method as many checks are not required in that case to avoid - // performance issues. - // Processing 82645 items took now 61 ms compared to earlier version where it took ages (> 2min). - // Usually we only get about a few hundred or max. a few 1000 items. 82645 is all - // trade stats stats and all account age witness data. - - // We only apply it once from first response - if (!initialRequestApplied) { - dataStorage.addPersistableNetworkPayloadFromInitialRequest(e); - - } - } else { - // We don't broadcast here as we are only connected to the seed node and would be pointless - dataStorage.addPersistableNetworkPayload(e, sender, false, - false, false, false); - } - }); - - // We set initialRequestApplied to true after the loop, otherwise we would only process 1 entry - initialRequestApplied = true; - - log.info("Processing {} persistableNetworkPayloads took {} ms.", - persistableNetworkPayloadSet.size(), System.currentTimeMillis() - ts2); - } + dataStorage.processGetDataResponse(getDataResponse, + connection.getPeersNodeAddressOptional().get()); cleanup(); listener.onComplete(); @@ -310,24 +246,20 @@ private void logContents(NetworkEnvelope networkEnvelope, payloadByClassName.get(className).add(protectedStoragePayload); }); + persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { + // For logging different data types + String className = persistableNetworkPayload.getClass().getSimpleName(); + if (!payloadByClassName.containsKey(className)) + payloadByClassName.put(className, new HashSet<>()); - if (persistableNetworkPayloadSet != null) { - persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { - // For logging different data types - String className = persistableNetworkPayload.getClass().getSimpleName(); - if (!payloadByClassName.containsKey(className)) - payloadByClassName.put(className, new HashSet<>()); - - payloadByClassName.get(className).add(persistableNetworkPayload); - }); - } + payloadByClassName.get(className).add(persistableNetworkPayload); + }); // Log different data types StringBuilder sb = new StringBuilder(); sb.append("\n#################################################################\n"); sb.append("Connected to node: " + peersNodeAddress.getFullAddress() + "\n"); - final int items = dataSet.size() + - (persistableNetworkPayloadSet != null ? persistableNetworkPayloadSet.size() : 0); + final int items = dataSet.size() + persistableNetworkPayloadSet.size(); sb.append("Received ").append(items).append(" instances\n"); payloadByClassName.forEach((key, value) -> sb.append(key) .append(": ") diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java index 0386d1abe1b..a86d6f27834 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java @@ -29,7 +29,6 @@ import bisq.common.proto.network.NetworkProtoResolver; import java.util.HashSet; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -37,7 +36,7 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; -import javax.annotation.Nullable; +import org.jetbrains.annotations.NotNull; @Slf4j @EqualsAndHashCode(callSuper = true) @@ -47,17 +46,15 @@ public final class GetDataResponse extends NetworkEnvelope implements SupportedC private final Set dataSet; // Set of PersistableNetworkPayload objects - // We added that in v 0.6 and we would get a null object from older peers, so keep it annotated with @Nullable - @Nullable + // We added that in v 0.6 and the fromProto code will create an empty HashSet if it doesn't exist private final Set persistableNetworkPayloadSet; private final int requestNonce; private final boolean isGetUpdatedDataResponse; - @Nullable private final Capabilities supportedCapabilities; - public GetDataResponse(Set dataSet, - @Nullable Set persistableNetworkPayloadSet, + public GetDataResponse(@NotNull Set dataSet, + @NotNull Set persistableNetworkPayloadSet, int requestNonce, boolean isGetUpdatedDataResponse) { this(dataSet, @@ -72,11 +69,11 @@ public GetDataResponse(Set dataSet, // PROTO BUFFER /////////////////////////////////////////////////////////////////////////////////////////// - private GetDataResponse(Set dataSet, - @Nullable Set persistableNetworkPayloadSet, + private GetDataResponse(@NotNull Set dataSet, + @NotNull Set persistableNetworkPayloadSet, int requestNonce, boolean isGetUpdatedDataResponse, - @Nullable Capabilities supportedCapabilities, + @NotNull Capabilities supportedCapabilities, int messageVersion) { super(messageVersion); @@ -100,13 +97,12 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { .setProtectedStorageEntry((protobuf.ProtectedStorageEntry) protectedStorageEntry.toProtoMessage()) .build()) .collect(Collectors.toList())) + .addAllPersistableNetworkPayloadItems(persistableNetworkPayloadSet.stream() + .map(PersistableNetworkPayload::toProtoMessage) + .collect(Collectors.toList())) .setRequestNonce(requestNonce) - .setIsGetUpdatedDataResponse(isGetUpdatedDataResponse); - - Optional.ofNullable(supportedCapabilities).ifPresent(e -> builder.addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities))); - Optional.ofNullable(persistableNetworkPayloadSet).ifPresent(set -> builder.addAllPersistableNetworkPayloadItems(set.stream() - .map(PersistableNetworkPayload::toProtoMessage) - .collect(Collectors.toList()))); + .setIsGetUpdatedDataResponse(isGetUpdatedDataResponse) + .addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities)); protobuf.NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setGetDataResponse(builder) @@ -124,14 +120,11 @@ public static GetDataResponse fromProto(protobuf.GetDataResponse proto, .map(entry -> (ProtectedStorageEntry) resolver.fromProto(entry)) .collect(Collectors.toSet())); - Set persistableNetworkPayloadSet = proto.getPersistableNetworkPayloadItemsList().isEmpty() ? - null : - new HashSet<>( - proto.getPersistableNetworkPayloadItemsList().stream() + Set persistableNetworkPayloadSet = new HashSet<>( + proto.getPersistableNetworkPayloadItemsList().stream() .map(e -> (PersistableNetworkPayload) resolver.fromProto(e)) .collect(Collectors.toSet())); - //PersistableNetworkPayload return new GetDataResponse(dataSet, persistableNetworkPayloadSet, proto.getRequestNonce(), diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java index 0702b0f1756..db0e51eb7f6 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java @@ -26,7 +26,6 @@ import com.google.protobuf.ByteString; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -34,9 +33,7 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; -import javax.annotation.Nullable; - - +import org.jetbrains.annotations.NotNull; import protobuf.NetworkEnvelope; @@ -44,11 +41,10 @@ @EqualsAndHashCode(callSuper = true) @Value public final class PreliminaryGetDataRequest extends GetDataRequest implements AnonymousMessage, SupportedCapabilitiesMessage { - @Nullable private final Capabilities supportedCapabilities; public PreliminaryGetDataRequest(int nonce, - Set excludedKeys) { + @NotNull Set excludedKeys) { this(nonce, excludedKeys, Capabilities.app, Version.getP2PMessageVersion()); } @@ -58,8 +54,8 @@ public PreliminaryGetDataRequest(int nonce, /////////////////////////////////////////////////////////////////////////////////////////// private PreliminaryGetDataRequest(int nonce, - Set excludedKeys, - @Nullable Capabilities supportedCapabilities, + @NotNull Set excludedKeys, + @NotNull Capabilities supportedCapabilities, int messageVersion) { super(messageVersion, nonce, excludedKeys); @@ -69,13 +65,12 @@ private PreliminaryGetDataRequest(int nonce, @Override public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { final protobuf.PreliminaryGetDataRequest.Builder builder = protobuf.PreliminaryGetDataRequest.newBuilder() + .addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities)) .setNonce(nonce) .addAllExcludedKeys(excludedKeys.stream() .map(ByteString::copyFrom) .collect(Collectors.toList())); - Optional.ofNullable(supportedCapabilities).ifPresent(e -> builder.addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities))); - NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setPreliminaryGetDataRequest(builder) .build(); @@ -85,13 +80,9 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { public static PreliminaryGetDataRequest fromProto(protobuf.PreliminaryGetDataRequest proto, int messageVersion) { log.info("Received a PreliminaryGetDataRequest with {} kB", proto.getSerializedSize() / 1000d); - Capabilities supportedCapabilities = proto.getSupportedCapabilitiesList().isEmpty() ? - null : - Capabilities.fromIntList(proto.getSupportedCapabilitiesList()); - return new PreliminaryGetDataRequest(proto.getNonce(), ProtoUtil.byteSetFromProtoByteStringList(proto.getExcludedKeysList()), - supportedCapabilities, + Capabilities.fromIntList(proto.getSupportedCapabilitiesList()), messageVersion); } } diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index c0a22c691b2..551b15bae49 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -25,6 +25,10 @@ import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.peers.BroadcastHandler; import bisq.network.p2p.peers.Broadcaster; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; import bisq.network.p2p.storage.messages.AddDataMessage; import bisq.network.p2p.storage.messages.AddOncePayload; import bisq.network.p2p.storage.messages.AddPersistableNetworkPayloadMessage; @@ -32,8 +36,10 @@ import bisq.network.p2p.storage.messages.RefreshOfferMessage; import bisq.network.p2p.storage.messages.RemoveDataMessage; import bisq.network.p2p.storage.messages.RemoveMailboxDataMessage; +import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; import bisq.network.p2p.storage.payload.DateTolerantPayload; import bisq.network.p2p.storage.payload.ExpirablePayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.MailboxStoragePayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedMailboxStorageEntry; @@ -48,6 +54,7 @@ import bisq.common.Timer; import bisq.common.UserThread; +import bisq.common.app.Capabilities; import bisq.common.crypto.CryptoException; import bisq.common.crypto.Hash; import bisq.common.crypto.Sig; @@ -88,6 +95,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; @@ -108,6 +118,8 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers @VisibleForTesting public static int CHECK_TTL_INTERVAL_SEC = 60; + private boolean initialRequestApplied = false; + private final Broadcaster broadcaster; private final AppendOnlyDataStoreService appendOnlyDataStoreService; private final ProtectedDataStoreService protectedDataStoreService; @@ -177,6 +189,180 @@ public synchronized void readFromResources(String postFix) { map.putAll(protectedDataStoreService.getMap()); } + /////////////////////////////////////////////////////////////////////////////////////////// + // RequestData API + /////////////////////////////////////////////////////////////////////////////////////////// + + /** + * Returns a PreliminaryGetDataRequest that can be sent to a peer node to request missing Payload data. + */ + public PreliminaryGetDataRequest buildPreliminaryGetDataRequest(int nonce) { + return new PreliminaryGetDataRequest(nonce, this.getKnownPayloadHashes()); + } + + /** + * Returns a GetUpdatedDataRequest that can be sent to a peer node to request missing Payload data. + */ + public GetUpdatedDataRequest buildGetUpdatedDataRequest(NodeAddress senderNodeAddress, int nonce) { + return new GetUpdatedDataRequest(senderNodeAddress, nonce, this.getKnownPayloadHashes()); + } + + /** + * Returns the set of known payload hashes. This is used in the GetData path to request missing data from peer nodes + */ + private Set getKnownPayloadHashes() { + // We collect the keys of the PersistableNetworkPayload items so we exclude them in our request. + // PersistedStoragePayload items don't get removed, so we don't have an issue with the case that + // an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would + // miss that event if we do not load the full set or use some delta handling. + Set excludedKeys =this.appendOnlyDataStoreService.getMap().keySet().stream() + .map(e -> e.bytes) + .collect(Collectors.toSet()); + + Set excludedKeysFromPersistedEntryMap = this.map.keySet() + .stream() + .map(e -> e.bytes) + .collect(Collectors.toSet()); + + excludedKeys.addAll(excludedKeysFromPersistedEntryMap); + + return excludedKeys; + } + + /** + * Generic function that can be used to filter a Map + * by a given set of keys and peer capabilities. + */ + static private Set filterKnownHashes( + Map toFilter, + Function objToPayload, + Set knownHashes, + Capabilities peerCapabilities, + int maxEntries, + AtomicBoolean outTruncated) { + + AtomicInteger limit = new AtomicInteger(maxEntries); + + Set filteredResults = toFilter.entrySet().stream() + .filter(e -> !knownHashes.contains(e.getKey())) + .filter(e -> limit.decrementAndGet() >= 0) + .map(Map.Entry::getValue) + .filter(networkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, + objToPayload.apply(networkPayload))) + .collect(Collectors.toSet()); + + if (limit.get() < 0) + outTruncated.set(true); + + return filteredResults; + } + + /** + * Returns a GetDataResponse object that contains the Payloads known locally, but not remotely. + */ + public GetDataResponse buildGetDataResponse( + GetDataRequest getDataRequest, + int maxEntriesPerType, + AtomicBoolean outPersistableNetworkPayloadOutputTruncated, + AtomicBoolean outProtectedStorageEntryOutputTruncated, + Capabilities peerCapabilities) { + + Set excludedKeysAsByteArray = + P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); + + Set filteredPersistableNetworkPayloads = + filterKnownHashes( + this.appendOnlyDataStoreService.getMap(), + Function.identity(), + excludedKeysAsByteArray, + peerCapabilities, + maxEntriesPerType, + outPersistableNetworkPayloadOutputTruncated); + + Set filteredProtectedStorageEntries = + filterKnownHashes( + this.map, + ProtectedStorageEntry::getProtectedStoragePayload, + excludedKeysAsByteArray, + peerCapabilities, + maxEntriesPerType, + outProtectedStorageEntryOutputTruncated); + + return new GetDataResponse( + filteredProtectedStorageEntries, + filteredPersistableNetworkPayloads, + getDataRequest.getNonce(), + getDataRequest instanceof GetUpdatedDataRequest); + } + + /** + * Returns true if a Payload should be transmit to a peer given the peer's supported capabilities. + */ + private static boolean shouldTransmitPayloadToPeer(Capabilities peerCapabilities, NetworkPayload payload) { + + // Sanity check to ensure this isn't used outside P2PDataStorage + if (!(payload instanceof ProtectedStoragePayload || payload instanceof PersistableNetworkPayload)) + return false; + + // If the payload doesn't have a required capability, we should transmit it + if (!(payload instanceof CapabilityRequiringPayload)) + return true; + + // Otherwise, only transmit the Payload if the peer supports all capabilities required by the payload + boolean shouldTransmit = peerCapabilities.containsAll(((CapabilityRequiringPayload) payload).getRequiredCapabilities()); + + if (!shouldTransmit) { + log.debug("We do not send the message to the peer because they do not support the required capability for that message type.\n" + + "storagePayload is: " + Utilities.toTruncatedString(payload)); + } + + return shouldTransmit; + } + + /** + * Processes a GetDataResponse message and updates internal state. Does not broadcast updates to the P2P network + * or domain listeners. + */ + public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress sender) { + final Set dataSet = getDataResponse.getDataSet(); + Set persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet(); + + long ts2 = System.currentTimeMillis(); + dataSet.forEach(e -> { + // We don't broadcast here (last param) as we are only connected to the seed node and would be pointless + addProtectedStorageEntry(e, sender, null, false, false); + + }); + log.info("Processing {} protectedStorageEntries took {} ms.", dataSet.size(), this.clock.millis() - ts2); + + ts2 = this.clock.millis(); + persistableNetworkPayloadSet.forEach(e -> { + if (e instanceof ProcessOncePersistableNetworkPayload) { + // We use an optimized method as many checks are not required in that case to avoid + // performance issues. + // Processing 82645 items took now 61 ms compared to earlier version where it took ages (> 2min). + // Usually we only get about a few hundred or max. a few 1000 items. 82645 is all + // trade stats stats and all account age witness data. + + // We only apply it once from first response + if (!initialRequestApplied) { + addPersistableNetworkPayloadFromInitialRequest(e); + + } + } else { + // We don't broadcast here as we are only connected to the seed node and would be pointless + addPersistableNetworkPayload(e, sender, false, + false, false, false); + } + }); + log.info("Processing {} persistableNetworkPayloads took {} ms.", + persistableNetworkPayloadSet.size(), this.clock.millis() - ts2); + + // We only process PersistableNetworkPayloads implementing ProcessOncePersistableNetworkPayload once. It can cause performance + // issues and since the data is rarely out of sync it is not worth it to apply them from multiple peers during + // startup. + initialRequestApplied = true; + } /////////////////////////////////////////////////////////////////////////////////////////// // API diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/LazyProcessedPayload.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProcessOncePersistableNetworkPayload.java similarity index 72% rename from p2p/src/main/java/bisq/network/p2p/storage/payload/LazyProcessedPayload.java rename to p2p/src/main/java/bisq/network/p2p/storage/payload/ProcessOncePersistableNetworkPayload.java index f63d1cf9b62..70fe1c35abe 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/LazyProcessedPayload.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProcessOncePersistableNetworkPayload.java @@ -20,8 +20,9 @@ import bisq.common.Payload; /** - * Marker interface for payload which gets delayed processed at startup so we don't hit performance too much. - * Used for TradeStatistics and AccountAgeWitness. + * Marker interface for PersistableNetworkPayloads that are only added during the FIRST call to + * P2PDataStorage::processDataResponse. This improves performance for objects that don't go out + * of sync frequently. */ -public interface LazyProcessedPayload extends Payload { +public interface ProcessOncePersistableNetworkPayload extends Payload { } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java new file mode 100644 index 00000000000..173d3358041 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java @@ -0,0 +1,481 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; +import bisq.common.crypto.Sig; + +import com.google.protobuf.Message; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + + + +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class P2PDataStorageBuildGetDataResponseTest { + abstract static class P2PDataStorageBuildGetDataResponseTestBase { + // GIVEN null & non-null supportedCapabilities + private TestState testState; + + abstract GetDataRequest buildGetDataRequest(int nonce, Set knownKeys); + + @Mock + NetworkNode networkNode; + + private NodeAddress localNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + + this.localNodeAddress = new NodeAddress("localhost", 8080); + when(networkNode.getNodeAddress()).thenReturn(this.localNodeAddress); + + // Set up basic capabilities to ensure message contains it + Capabilities.app.addAll(Capability.MEDIATION); + } + + static class RequiredCapabilitiesPNPStub extends PersistableNetworkPayloadStub + implements CapabilityRequiringPayload { + Capabilities capabilities; + + RequiredCapabilitiesPNPStub(Capabilities capabilities, byte[] hash) { + super(hash); + this.capabilities = capabilities; + } + + @Override + public Capabilities getRequiredCapabilities() { + return capabilities; + } + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + return getProtectedStorageEntryForAdd(null); + } + + private ProtectedStorageEntry getProtectedStorageEntryForAdd(Capabilities requiredCapabilities) + throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + // Payload stub + ProtectedStoragePayload protectedStoragePayload; + + if (requiredCapabilities == null) + protectedStoragePayload = mock(ProtectedStoragePayload.class); + else { + protectedStoragePayload = mock(ProtectedStoragePayload.class, + withSettings().extraInterfaces(CapabilityRequiringPayload.class)); + when(((CapabilityRequiringPayload) protectedStoragePayload).getRequiredCapabilities()) + .thenReturn(requiredCapabilities); + } + + Message messageMock = mock(Message.class); + when(messageMock.toByteArray()).thenReturn(Sig.getPublicKeyBytes(ownerKeys.getPublic())); + when(protectedStoragePayload.toProtoMessage()).thenReturn(messageMock); + + // Entry stub + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + // TESTCASE: Given a GetDataRequest w/ unknown PNP, nothing is sent back + @Test + public void buildGetDataResponse_unknownPNPDoNothing() { + PersistableNetworkPayload fromPeer = new PersistableNetworkPayloadStub(new byte[]{1}); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>(Collections.singletonList(fromPeer.getHash()))); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/ known PNP, nothing is sent back + @Test + public void buildGetDataResponse_knownPNPDoNothing() { + PersistableNetworkPayload fromPeerAndLocal = new PersistableNetworkPayloadStub(new byte[]{1}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + fromPeerAndLocal, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest( + 1, + new HashSet<>(Collections.singletonList(fromPeerAndLocal.getHash()))); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, send it back + @Test + public void buildGetDataResponse_unknownPNPSendBack() { + PersistableNetworkPayload onlyLocal = new PersistableNetworkPayloadStub(new byte[]{1}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().contains(onlyLocal)); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, don't send more than truncation limit + @Test + public void buildGetDataResponse_unknownPNPSendBackTruncation() { + PersistableNetworkPayload onlyLocal1 = new PersistableNetworkPayloadStub(new byte[]{1}); + PersistableNetworkPayload onlyLocal2 = new PersistableNetworkPayloadStub(new byte[]{2}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal1, this.localNodeAddress, false, false, false, false); + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal2, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertTrue(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertEquals(1, getDataResponse.getPersistableNetworkPayloadSet().size()); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().contains(onlyLocal1)); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, but missing required capabilities, nothing is sent back + @Test + public void buildGetDataResponse_unknownPNPCapabilitiesMismatchDontSendBack() { + PersistableNetworkPayload onlyLocal = + new RequiredCapabilitiesPNPStub(new Capabilities(Collections.singletonList(Capability.MEDIATION)), + new byte[]{1}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP that requires capabilities (and they match) send it back + @Test + public void buildGetDataResponse_unknownPNPCapabilitiesMatch() { + PersistableNetworkPayload onlyLocal = + new RequiredCapabilitiesPNPStub(new Capabilities(Collections.singletonList(Capability.MEDIATION)), + new byte[]{1}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(Collections.singletonList(Capability.MEDIATION)); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().contains(onlyLocal)); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/ unknown PSE, nothing is sent back + @Test + public void buildGetDataResponse_unknownPSEDoNothing() throws NoSuchAlgorithmException { + ProtectedStorageEntry fromPeer = getProtectedStorageEntryForAdd(); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, + new HashSet<>(Collections.singletonList( + P2PDataStorage.get32ByteHash(fromPeer.getProtectedStoragePayload())))); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/ known PSE, nothing is sent back + @Test + public void buildGetDataResponse_knownPSEDoNothing() throws NoSuchAlgorithmException { + ProtectedStorageEntry fromPeerAndLocal = getProtectedStorageEntryForAdd(); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, + new HashSet<>(Collections.singletonList( + P2PDataStorage.get32ByteHash(fromPeerAndLocal.getProtectedStoragePayload())))); + + this.testState.mockedStorage.addProtectedStorageEntry( + fromPeerAndLocal, this.localNodeAddress, null, false); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PSE, send it back + @Test + public void buildGetDataResponse_unknownPSESendBack() throws NoSuchAlgorithmException { + ProtectedStorageEntry onlyLocal = getProtectedStorageEntryForAdd(); + + GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); + + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal, this.localNodeAddress, null, false); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().contains(onlyLocal)); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, don't send more than truncation limit + @Test + public void buildGetDataResponse_unknownPSESendBackTruncation() throws NoSuchAlgorithmException { + ProtectedStorageEntry onlyLocal1 = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry onlyLocal2 = getProtectedStorageEntryForAdd(); + + GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); + + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal1, this.localNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal2, this.localNodeAddress, null, false); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertTrue(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertEquals(1, getDataResponse.getDataSet().size()); + Assert.assertTrue( + getDataResponse.getDataSet().contains(onlyLocal1) + || getDataResponse.getDataSet().contains(onlyLocal2)); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, but missing required capabilities, nothing is sent back + @Test + public void buildGetDataResponse_unknownPSECapabilitiesMismatchDontSendBack() throws NoSuchAlgorithmException { + ProtectedStorageEntry onlyLocal = + getProtectedStorageEntryForAdd(new Capabilities(Collections.singletonList(Capability.MEDIATION))); + + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal, this.localNodeAddress, null, false); + + GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP that requires capabilities (and they match) send it back + @Test + public void buildGetDataResponse_unknownPSECapabilitiesMatch() throws NoSuchAlgorithmException { + ProtectedStorageEntry onlyLocal = + getProtectedStorageEntryForAdd(new Capabilities(Collections.singletonList(Capability.MEDIATION))); + + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal, this.localNodeAddress, null, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(Collections.singletonList(Capability.MEDIATION)); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().contains(onlyLocal)); + } + } + + public static class P2PDataStorageBuildGetDataResponseTestPreliminary extends P2PDataStorageBuildGetDataResponseTestBase { + + @Override + GetDataRequest buildGetDataRequest(int nonce, Set knownKeys) { + return new PreliminaryGetDataRequest(nonce, knownKeys); + } + } + + public static class P2PDataStorageBuildGetDataResponseTestUpdated extends P2PDataStorageBuildGetDataResponseTestBase { + + @Override + GetDataRequest buildGetDataRequest(int nonce, Set knownKeys) { + return new GetUpdatedDataRequest(new NodeAddress("peer", 10), nonce, knownKeys); + } + } +} diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java index 4e0cb6c895f..7126b61770a 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java @@ -73,7 +73,7 @@ public void getProtectedStorageEntry_NoExist() throws NoSuchAlgorithmException, SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } // TESTCASE: Adding an entry from the getProtectedStorageEntry API of an existing item correctly updates the item @@ -90,7 +90,7 @@ public void getProtectedStorageEntry() throws NoSuchAlgorithmException, CryptoEx protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } // TESTCASE: Adding an entry from the getProtectedStorageEntry API of an existing item (added from onMessage path) correctly updates the item @@ -110,7 +110,7 @@ public void getProtectedStorageEntry_FirstOnMessageSecondAPI() throws NoSuchAlgo protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } // TESTCASE: Updating an entry from the getRefreshTTLMessage API correctly errors if the item hasn't been seen diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java new file mode 100644 index 00000000000..3b40575c633 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java @@ -0,0 +1,193 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage; + +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.storage.mocks.PersistableExpirableProtectedStoragePayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.app.Capabilities; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Assert; +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class P2PDataStorageGetDataIntegrationTest { + + /** + * Generates a unique ProtectedStorageEntry that is valid for add and remove. + */ + private ProtectedStorageEntry getProtectedStorageEntry() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + return getProtectedStorageEntry( + ownerKeys.getPublic(), new ProtectedStoragePayloadStub(ownerKeys.getPublic()), 1); + } + + private ProtectedStorageEntry getProtectedStorageEntry( + PublicKey ownerPubKey, + ProtectedStoragePayload protectedStoragePayload, + int sequenceNumber) { + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerPubKey); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.isValidForRemoveOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(sequenceNumber); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + // TESTCASE: Basic synchronization of a ProtectedStorageEntry works between a seed node and client node + @Test + public void basicSynchronizationWorks() throws NoSuchAlgorithmException { + TestState seedNodeTestState = new TestState(); + P2PDataStorage seedNode = seedNodeTestState.mockedStorage; + + TestState clientNodeTestState = new TestState(); + P2PDataStorage clientNode = clientNodeTestState.mockedStorage; + + ProtectedStorageEntry onSeedNode = getProtectedStorageEntry(); + seedNode.addProtectedStorageEntry(onSeedNode, null, null, false); + + GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); + + GetDataResponse getDataResponse = seedNode.buildGetDataResponse( + getDataRequest, 1, new AtomicBoolean(), new AtomicBoolean(), new Capabilities()); + + TestState.SavedTestState beforeState = clientNodeTestState.saveTestState(onSeedNode); + clientNode.processGetDataResponse(getDataResponse, null); + + clientNodeTestState.verifyProtectedStorageAdd( + beforeState, onSeedNode, true, true, false, true, false); + } + + // TESTCASE: Synchronization after peer restart works for in-memory ProtectedStorageEntrys + @Test + public void basicSynchronizationWorksAfterRestartTransient() throws NoSuchAlgorithmException { + ProtectedStorageEntry transientEntry = getProtectedStorageEntry(); + + TestState seedNodeTestState = new TestState(); + P2PDataStorage seedNode = seedNodeTestState.mockedStorage; + + TestState clientNodeTestState = new TestState(); + P2PDataStorage clientNode = clientNodeTestState.mockedStorage; + + seedNode.addProtectedStorageEntry(transientEntry, null, null, false); + + clientNode.addProtectedStorageEntry(transientEntry, null, null, false); + + clientNodeTestState.simulateRestart(); + clientNode = clientNodeTestState.mockedStorage; + + GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); + + GetDataResponse getDataResponse = seedNode.buildGetDataResponse( + getDataRequest, 1, new AtomicBoolean(), new AtomicBoolean(), new Capabilities()); + + TestState.SavedTestState beforeState = clientNodeTestState.saveTestState(transientEntry); + clientNode.processGetDataResponse(getDataResponse, null); + + clientNodeTestState.verifyProtectedStorageAdd( + beforeState, transientEntry, true, true, false, true, false); + } + + // TESTCASE: Synchronization after peer restart works for in-memory ProtectedStorageEntrys + @Test + public void basicSynchronizationWorksAfterRestartPersistent() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + ProtectedStoragePayload persistentPayload = + new PersistableExpirableProtectedStoragePayloadStub(ownerKeys.getPublic()); + ProtectedStorageEntry persistentEntry = getProtectedStorageEntry(ownerKeys.getPublic(), persistentPayload, 1); + + TestState seedNodeTestState = new TestState(); + P2PDataStorage seedNode = seedNodeTestState.mockedStorage; + + TestState clientNodeTestState = new TestState(); + P2PDataStorage clientNode = clientNodeTestState.mockedStorage; + + seedNode.addProtectedStorageEntry(persistentEntry, null, null, false); + + clientNode.addProtectedStorageEntry(persistentEntry, null, null, false); + + clientNodeTestState.simulateRestart(); + clientNode = clientNodeTestState.mockedStorage; + + GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); + + GetDataResponse getDataResponse = seedNode.buildGetDataResponse( + getDataRequest, 1, new AtomicBoolean(), new AtomicBoolean(), new Capabilities()); + + TestState.SavedTestState beforeState = clientNodeTestState.saveTestState(persistentEntry); + clientNode.processGetDataResponse(getDataResponse, null); + + clientNodeTestState.verifyProtectedStorageAdd( + beforeState, persistentEntry, false, false, false, false, false); + Assert.assertTrue(clientNodeTestState.mockedStorage.getMap().containsValue(persistentEntry)); + } + + // TESTCASE: Removes seen only by the seednode should be replayed on the client node + // during startup + // XXXBUGXXX: #3610 Lost removes are never replayed. + @Test + public void lostRemoveNeverUpdated() throws NoSuchAlgorithmException { + TestState seedNodeTestState = new TestState(); + P2PDataStorage seedNode = seedNodeTestState.mockedStorage; + + TestState clientNodeTestState = new TestState(); + P2PDataStorage clientNode = clientNodeTestState.mockedStorage; + + // Both nodes see the add + KeyPair ownerKeys = TestUtils.generateKeyPair(); + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + ProtectedStorageEntry onSeedNodeAndClientNode = getProtectedStorageEntry( + ownerKeys.getPublic(), protectedStoragePayload, 1); + seedNode.addProtectedStorageEntry(onSeedNodeAndClientNode, null, null, false); + clientNode.addProtectedStorageEntry(onSeedNodeAndClientNode, null, null, false); + + // Seed node sees the remove, but client node does not + seedNode.remove(getProtectedStorageEntry( + ownerKeys.getPublic(), protectedStoragePayload, 2), null, false); + + GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); + + GetDataResponse getDataResponse = seedNode.buildGetDataResponse( + getDataRequest, 1, new AtomicBoolean(), new AtomicBoolean(), new Capabilities()); + + TestState.SavedTestState beforeState = clientNodeTestState.saveTestState(onSeedNodeAndClientNode); + clientNode.processGetDataResponse(getDataResponse, null); + + // Should succeed + clientNodeTestState.verifyProtectedStorageRemove( + beforeState, onSeedNodeAndClientNode, false, false, false, false, false); + } +} diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java index 9213bcb0fbe..45c1ce17689 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java @@ -107,7 +107,9 @@ void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean testState.mockedStorage.onMessage(new AddPersistableNetworkPayloadMessage(persistableNetworkPayload), mockedConnection); } - this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, this.expectBroadcastOnStateChange(), this.expectedIsDataOwner()); + boolean expectedBroadcast = expectedStateChange && this.expectBroadcastOnStateChange(); + + this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, expectedBroadcast, expectedBroadcast, this.expectedIsDataOwner()); } @Before diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java new file mode 100644 index 00000000000..c7177b005d7 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java @@ -0,0 +1,263 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.mockito.MockitoAnnotations; + +public class P2PDataStorageProcessGetDataResponse { + private TestState testState; + + private NodeAddress peerNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + + this.peerNodeAddress = new NodeAddress("peer", 8080); + } + + static private GetDataResponse buildGetDataResponse(PersistableNetworkPayload persistableNetworkPayload) { + return buildGetDataResponse(Collections.emptyList(), Collections.singletonList(persistableNetworkPayload)); + } + + static private GetDataResponse buildGetDataResponse(ProtectedStorageEntry protectedStorageEntry) { + return buildGetDataResponse(Collections.singletonList(protectedStorageEntry), Collections.emptyList()); + } + + static private GetDataResponse buildGetDataResponse( + List protectedStorageEntries, + List persistableNetworkPayloads) { + return new GetDataResponse( + new HashSet<>(protectedStorageEntries), + new HashSet<>(persistableNetworkPayloads), + 1, + false); + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + static class LazyPersistableNetworkPayloadStub extends PersistableNetworkPayloadStub + implements ProcessOncePersistableNetworkPayload { + + LazyPersistableNetworkPayloadStub(byte[] hash) { + super(hash); + } + + LazyPersistableNetworkPayloadStub(boolean validHashSize) { + super(validHashSize); + } + } + + // TESTCASE: GetDataResponse w/ missing PNP is added with no broadcast or listener signal + // XXXBUGXXX: We signal listeners w/ non ProcessOncePersistableNetworkPayloads + @Test + public void processGetDataResponse_newPNPUpdatesState() { + PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(new byte[] { 1 }); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, true, true, false, false); + } + + // TESTCASE: GetDataResponse w/ invalid PNP does nothing (LazyProcessed) + @Test + public void processGetDataResponse_newInvalidPNPDoesNothing() { + PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(false); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ existing PNP changes no state + @Test + public void processGetDataResponse_duplicatePNPDoesNothing() { + PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(new byte[] { 1 }); + this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, + this.peerNodeAddress, false, false, false, false); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ missing PNP is added with no broadcast or listener signal (ProcessOncePersistableNetworkPayload) + @Test + public void processGetDataResponse_newPNPUpdatesState_LazyProcessed() { + PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, true, false, false, false); + } + + // TESTCASE: GetDataResponse w/ existing PNP changes no state (ProcessOncePersistableNetworkPayload) + @Test + public void processGetDataResponse_duplicatePNPDoesNothing_LazyProcessed() { + PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); + this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, + this.peerNodeAddress, false, false, false, false); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, false, false, false, false); + } + + // TESTCASE: Second call to processGetDataResponse adds PNP for non-ProcessOncePersistableNetworkPayloads + @Test + public void processGetDataResponse_secondProcessNewPNPUpdatesState() { + PersistableNetworkPayload addFromFirstProcess = new PersistableNetworkPayloadStub(new byte[] { 1 }); + GetDataResponse getDataResponse = buildGetDataResponse(addFromFirstProcess); + + TestState.SavedTestState beforeState = this.testState.saveTestState(addFromFirstProcess); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, addFromFirstProcess, true, true, false, false); + + PersistableNetworkPayload addFromSecondProcess = new PersistableNetworkPayloadStub(new byte[] { 2 }); + getDataResponse = buildGetDataResponse(addFromSecondProcess); + beforeState = this.testState.saveTestState(addFromSecondProcess); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, addFromSecondProcess, true, true, false, false); + } + + // TESTCASE: Second call to processGetDataResponse does not add any PNP (LazyProcessed) + @Test + public void processGetDataResponse_secondProcessNoPNPUpdates_LazyProcessed() { + PersistableNetworkPayload addFromFirstProcess = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); + GetDataResponse getDataResponse = buildGetDataResponse(addFromFirstProcess); + + TestState.SavedTestState beforeState = this.testState.saveTestState(addFromFirstProcess); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, addFromFirstProcess, true, false, false, false); + + PersistableNetworkPayload addFromSecondProcess = new LazyPersistableNetworkPayloadStub(new byte[] { 2 }); + getDataResponse = buildGetDataResponse(addFromSecondProcess); + beforeState = this.testState.saveTestState(addFromSecondProcess); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, addFromSecondProcess, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ missing PSE is added with no broadcast or listener signal + // XXXBUGXXX: We signal listeners for all ProtectedStorageEntrys + @Test + public void processGetDataResponse_newPSEUpdatesState() throws NoSuchAlgorithmException { + ProtectedStorageEntry protectedStorageEntry = getProtectedStorageEntryForAdd(); + GetDataResponse getDataResponse = buildGetDataResponse(protectedStorageEntry); + + TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, true, true, false, true, false); + } + + // TESTCASE: GetDataResponse w/ existing PSE changes no state + @Test + public void processGetDataResponse_duplicatePSEDoesNothing() throws NoSuchAlgorithmException { + ProtectedStorageEntry protectedStorageEntry = getProtectedStorageEntryForAdd(); + this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, this.peerNodeAddress, null, false); + + GetDataResponse getDataResponse = buildGetDataResponse(protectedStorageEntry); + + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, false, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ missing PSE is added with no broadcast or listener signal + // XXXBUGXXX: We signal listeners for all ProtectedStorageEntrys + @Test + public void processGetDataResponse_secondCallNewPSEUpdatesState() throws NoSuchAlgorithmException { + ProtectedStorageEntry protectedStorageEntry = getProtectedStorageEntryForAdd(); + GetDataResponse getDataResponse = buildGetDataResponse(protectedStorageEntry); + + TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, true, true, false, true, false); + + protectedStorageEntry = getProtectedStorageEntryForAdd(); + getDataResponse = buildGetDataResponse(protectedStorageEntry); + beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, true, true, false, true, false); + } +} diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java index 9f5bd539234..04eb291f309 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java @@ -197,7 +197,13 @@ void doProtectedStorageAddAndVerify(ProtectedStorageEntry protectedStorageEntry, if (!this.useMessageHandler) Assert.assertEquals(expectedReturnValue, addResult); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, expectedStateChange, this.expectIsDataOwner()); + if (expectedStateChange) { + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, true, true, true, true, this.expectIsDataOwner()); + } else{ + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, false, false, false, false, this.expectIsDataOwner()); + } } void doProtectedStorageRemoveAndVerify(ProtectedStorageEntry entry, diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java new file mode 100644 index 00000000000..55d16344d64 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java @@ -0,0 +1,175 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Set; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.*; + +public class P2PDataStorageRequestDataTest { + private TestState testState; + + private NodeAddress localNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + + this.localNodeAddress = new NodeAddress("localhost", 8080); + + // Set up basic capabilities to ensure message contains it + Capabilities.app.addAll(Capability.MEDIATION); + } + + /** + * Returns true if the target bytes are found in the container set. + */ + private boolean byteSetContains(Set container, byte[] target) { + // Set.contains() doesn't do a deep compare, so generate a Set so equals() does what + // we want + Set translatedContainer = + P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(container); + + return translatedContainer.contains(new P2PDataStorage.ByteArray(target)); + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + // TESTCASE: P2PDataStorage with no entries returns an empty PreliminaryGetDataRequest + @Test + public void buildPreliminaryGetDataRequest_EmptyP2PDataStore() { + PreliminaryGetDataRequest getDataRequest = this.testState.mockedStorage.buildPreliminaryGetDataRequest(1); + + Assert.assertEquals(getDataRequest.getNonce(), 1); + Assert.assertEquals(getDataRequest.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataRequest.getExcludedKeys().isEmpty()); + } + + // TESTCASE: P2PDataStorage with no entries returns an empty PreliminaryGetDataRequest + @Test + public void buildGetUpdatedDataRequest_EmptyP2PDataStore() { + GetUpdatedDataRequest getDataRequest = + this.testState.mockedStorage.buildGetUpdatedDataRequest(this.localNodeAddress, 1); + + Assert.assertEquals(getDataRequest.getNonce(), 1); + Assert.assertEquals(getDataRequest.getSenderNodeAddress(), this.localNodeAddress); + Assert.assertTrue(getDataRequest.getExcludedKeys().isEmpty()); + } + + // TESTCASE: P2PDataStorage with PersistableNetworkPayloads and ProtectedStorageEntry generates + // correct GetDataRequestMessage with both sets of keys. + @Test + public void buildPreliminaryGetDataRequest_FilledP2PDataStore() throws NoSuchAlgorithmException { + PersistableNetworkPayload toAdd1 = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload toAdd2 = new PersistableNetworkPayloadStub(new byte[] { 2 }); + ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); + + this.testState.mockedStorage.addPersistableNetworkPayload( + toAdd1, this.localNodeAddress, false, false, false, false); + this.testState.mockedStorage.addPersistableNetworkPayload( + toAdd2, this.localNodeAddress, false, false, false, false); + + this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); + + PreliminaryGetDataRequest getDataRequest = this.testState.mockedStorage.buildPreliminaryGetDataRequest(1); + + Assert.assertEquals(getDataRequest.getNonce(), 1); + Assert.assertEquals(getDataRequest.getSupportedCapabilities(), Capabilities.app); + Assert.assertEquals(4, getDataRequest.getExcludedKeys().size()); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd1.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd2.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd3.getProtectedStoragePayload()))); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd4.getProtectedStoragePayload()))); + } + + // TESTCASE: P2PDataStorage with PersistableNetworkPayloads and ProtectedStorageEntry generates + // correct GetDataRequestMessage with both sets of keys. + @Test + public void requestData_FilledP2PDataStore_GetUpdatedDataRequest() throws NoSuchAlgorithmException { + PersistableNetworkPayload toAdd1 = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload toAdd2 = new PersistableNetworkPayloadStub(new byte[] { 2 }); + ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); + + this.testState.mockedStorage.addPersistableNetworkPayload( + toAdd1, this.localNodeAddress, false, false, false, false); + this.testState.mockedStorage.addPersistableNetworkPayload( + toAdd2, this.localNodeAddress, false, false, false, false); + + this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); + + GetUpdatedDataRequest getDataRequest = + this.testState.mockedStorage.buildGetUpdatedDataRequest(this.localNodeAddress, 1); + + Assert.assertEquals(getDataRequest.getNonce(), 1); + Assert.assertEquals(getDataRequest.getSenderNodeAddress(), this.localNodeAddress); + Assert.assertEquals(4, getDataRequest.getExcludedKeys().size()); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd1.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd2.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd3.getProtectedStoragePayload()))); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd4.getProtectedStoragePayload()))); + } +} diff --git a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java index c4ceab5ca2a..51d831df41a 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java @@ -171,52 +171,55 @@ private void verifySequenceNumberMapWriteContains(P2PDataStorage.ByteArray paylo void verifyPersistableAdd(SavedTestState beforeState, PersistableNetworkPayload persistableNetworkPayload, - boolean expectedStateChange, - boolean expectedBroadcastAndListenersSignaled, + boolean expectedHashMapAndDataStoreUpdated, + boolean expectedListenersSignaled, + boolean expectedBroadcast, boolean expectedIsDataOwner) { P2PDataStorage.ByteArray hash = new P2PDataStorage.ByteArray(persistableNetworkPayload.getHash()); - if (expectedStateChange) { - // Payload is accessible from get() + if (expectedHashMapAndDataStoreUpdated) Assert.assertEquals(persistableNetworkPayload, this.mockedStorage.getAppendOnlyDataStoreMap().get(hash)); - } else { - // On failure, just ensure the state remained the same as before the add - if (beforeState.persistableNetworkPayloadBeforeOp != null) - Assert.assertEquals(beforeState.persistableNetworkPayloadBeforeOp, this.mockedStorage.getAppendOnlyDataStoreMap().get(hash)); - else - Assert.assertNull(this.mockedStorage.getAppendOnlyDataStoreMap().get(hash)); - } - - if (expectedStateChange && expectedBroadcastAndListenersSignaled) { - // Broadcast Called - verify(this.mockBroadcaster).broadcast(any(AddPersistableNetworkPayloadMessage.class), any(NodeAddress.class), - eq(null), eq(expectedIsDataOwner)); + else + Assert.assertEquals(beforeState.persistableNetworkPayloadBeforeOp, this.mockedStorage.getAppendOnlyDataStoreMap().get(hash)); - // Verify the listeners were updated once + if (expectedListenersSignaled) verify(this.appendOnlyDataStoreListener).onAdded(persistableNetworkPayload); + else + verify(this.appendOnlyDataStoreListener, never()).onAdded(persistableNetworkPayload); - } else { + if (expectedBroadcast) + verify(this.mockBroadcaster).broadcast(any(AddPersistableNetworkPayloadMessage.class), any(NodeAddress.class), + eq(null), eq(expectedIsDataOwner)); + else verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class), anyBoolean()); - - // Verify the listeners were never updated - verify(this.appendOnlyDataStoreListener, never()).onAdded(persistableNetworkPayload); - } } void verifyProtectedStorageAdd(SavedTestState beforeState, ProtectedStorageEntry protectedStorageEntry, - boolean expectedStateChange, + boolean expectedHashMapAndDataStoreUpdated, + boolean expectedListenersSignaled, + boolean expectedBroadcast, + boolean expectedSequenceNrMapWrite, boolean expectedIsDataOwner) { P2PDataStorage.ByteArray hashMapHash = P2PDataStorage.get32ByteHashAsByteArray(protectedStorageEntry.getProtectedStoragePayload()); - if (expectedStateChange) { + if (expectedHashMapAndDataStoreUpdated) { Assert.assertEquals(protectedStorageEntry, this.mockedStorage.getMap().get(hashMapHash)); if (protectedStorageEntry.getProtectedStoragePayload() instanceof PersistablePayload) Assert.assertEquals(protectedStorageEntry, this.protectedDataStoreService.getMap().get(hashMapHash)); + } else { + Assert.assertEquals(beforeState.protectedStorageEntryBeforeOp, this.mockedStorage.getMap().get(hashMapHash)); + Assert.assertEquals(beforeState.protectedStorageEntryBeforeOpDataStoreMap, this.protectedDataStoreService.getMap().get(hashMapHash)); + } + if (expectedListenersSignaled) { verify(this.hashMapChangedListener).onAdded(Collections.singletonList(protectedStorageEntry)); + } else { + verify(this.hashMapChangedListener, never()).onAdded(Collections.singletonList(protectedStorageEntry)); + } + if (expectedBroadcast) { final ArgumentCaptor captor = ArgumentCaptor.forClass(BroadcastMessage.class); verify(this.mockBroadcaster).broadcast(captor.capture(), any(NodeAddress.class), eq(null), eq(expectedIsDataOwner)); @@ -224,16 +227,13 @@ void verifyProtectedStorageAdd(SavedTestState beforeState, BroadcastMessage broadcastMessage = captor.getValue(); Assert.assertTrue(broadcastMessage instanceof AddDataMessage); Assert.assertEquals(protectedStorageEntry, ((AddDataMessage) broadcastMessage).getProtectedStorageEntry()); - - this.verifySequenceNumberMapWriteContains(P2PDataStorage.get32ByteHashAsByteArray(protectedStorageEntry.getProtectedStoragePayload()), protectedStorageEntry.getSequenceNumber()); } else { - Assert.assertEquals(beforeState.protectedStorageEntryBeforeOp, this.mockedStorage.getMap().get(hashMapHash)); - Assert.assertEquals(beforeState.protectedStorageEntryBeforeOpDataStoreMap, this.protectedDataStoreService.getMap().get(hashMapHash)); - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class), anyBoolean()); + } - // Internal state didn't change... nothing should be notified - verify(this.hashMapChangedListener, never()).onAdded(Collections.singletonList(protectedStorageEntry)); + if (expectedSequenceNrMapWrite) { + this.verifySequenceNumberMapWriteContains(P2PDataStorage.get32ByteHashAsByteArray(protectedStorageEntry.getProtectedStoragePayload()), protectedStorageEntry.getSequenceNumber()); + } else { verify(this.mockSeqNrStorage, never()).queueUpForSave(any(SequenceNumberMap.class), anyLong()); } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/mocks/PersistableNetworkPayloadStub.java b/p2p/src/test/java/bisq/network/p2p/storage/mocks/PersistableNetworkPayloadStub.java index 6ab73155480..057cb2c42b8 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/mocks/PersistableNetworkPayloadStub.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/mocks/PersistableNetworkPayloadStub.java @@ -28,9 +28,19 @@ */ public class PersistableNetworkPayloadStub implements PersistableNetworkPayload { private final boolean hashSizeValid; + private final byte[] hash; public PersistableNetworkPayloadStub(boolean hashSizeValid) { + this(hashSizeValid, new byte[] { 1 }); + } + + public PersistableNetworkPayloadStub(byte[] hash) { + this(true, hash); + } + + private PersistableNetworkPayloadStub(boolean hashSizeValid, byte[] hash) { this.hashSizeValid = hashSizeValid; + this.hash = hash; } @Override @@ -40,7 +50,7 @@ public protobuf.PersistableNetworkPayload toProtoMessage() { @Override public byte[] getHash() { - return new byte[] { 1 }; + return hash; } @Override