[NC-875] Implement iterative peer search

ethereum/p2p/src/main/java/tech/pegasys/pantheon/ethereum/p2p/discovery/internal/PeerDistanceCalculator.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2018 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.ethereum.p2p.discovery.internal;
+
+import tech.pegasys.pantheon.util.bytes.BytesValue;
+
+import java.util.Arrays;
+
+public class PeerDistanceCalculator {
+
+  /**
+   * Calculates the XOR distance between two values.
+   *
+   * @param v1 the first value
+   * @param v2 the second value
+   * @return the distance
+   */
+  static int distance(final BytesValue v1, final BytesValue v2) {
+    assert (v1.size() == v2.size());
+    final byte[] v1b = v1.extractArray();
+    final byte[] v2b = v2.extractArray();
+    if (Arrays.equals(v1b, v2b)) {
+      return 0;
+    }
+    int distance = v1b.length * 8;
+    for (int i = 0; i < v1b.length; i++) {
+      final byte xor = (byte) (0xff & (v1b[i] ^ v2b[i]));
+      if (xor == 0) {
+        distance -= 8;
+      } else {
+        int p = 7;
+        while (((xor >> p--) & 0x01) == 0) {
+          distance--;
+        }
+        break;
+      }
+    }
+    return distance;
+  }
+}

ethereum/p2p/src/main/java/tech/pegasys/pantheon/ethereum/p2p/discovery/internal/PeerTable.java

@@ -15,6 +15,7 @@
 import static java.util.Collections.unmodifiableList;
 import static java.util.Comparator.comparingInt;
 import static java.util.stream.Collectors.toList;
+import static tech.pegasys.pantheon.ethereum.p2p.discovery.internal.PeerDistanceCalculator.distance;
 
 import tech.pegasys.pantheon.crypto.Hash;
 import tech.pegasys.pantheon.ethereum.p2p.discovery.DiscoveryPeer;
@@ -203,38 +204,6 @@ private int distanceFrom(final PeerId peer) {
     return distance == null ? distance(keccak256, peer.keccak256()) : distance;
   }
 
-  /**
-   * Calculates the XOR distance between two values.
-   *
-   * @param v1 the first value
-   * @param v2 the second value
-   * @return the distance
-   */
-  static int distance(final BytesValue v1, final BytesValue v2) {
-    assert (v1.size() == v2.size());
-    final byte[] v1b = v1.extractArray();
-    final byte[] v2b = v2.extractArray();
-
-    if (Arrays.equals(v1b, v2b)) {
-      return 0;
-    }
-
-    int distance = v1b.length * 8;
-    for (int i = 0; i < v1b.length; i++) {
-      final byte xor = (byte) (0xff & (v1b[i] ^ v2b[i]));
-      if (xor == 0) {
-        distance -= 8;
-      } else {
-        int p = 7;
-        while (((xor >> p--) & 0x01) == 0) {
-          distance--;
-        }
-        break;
-      }
-    }
-    return distance;
-  }
-
   /** A class that encapsulates the result of a peer addition to the table. */
   public static class AddResult {
     /** The outcome of the operation. */

ethereum/p2p/src/main/java/tech/pegasys/pantheon/ethereum/p2p/discovery/internal/RecursivePeerRefreshState.java

@@ -0,0 +1,223 @@
+/*
+ * Copyright 2018 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.ethereum.p2p.discovery.internal;
+
+import static java.util.stream.Collectors.toList;
+import static tech.pegasys.pantheon.ethereum.p2p.discovery.internal.PeerDistanceCalculator.distance;
+
+import tech.pegasys.pantheon.ethereum.p2p.discovery.DiscoveryPeer;
+import tech.pegasys.pantheon.ethereum.p2p.peers.Peer;
+import tech.pegasys.pantheon.ethereum.p2p.peers.PeerBlacklist;
+import tech.pegasys.pantheon.util.bytes.BytesValue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import com.google.common.annotations.VisibleForTesting;
+
+class RecursivePeerRefreshState {
+  private final int CONCURRENT_REQUEST_LIMIT = 3;
+  private final BytesValue target;
+  private final PeerBlacklist peerBlacklist;
+  private final BondingAgent bondingAgent;
+  private final NeighborFinder neighborFinder;
+  private final List<PeerDistance> anteList;
+  private final List<OutstandingRequest> outstandingRequestList;
+  private final List<BytesValue> contactedInCurrentExecution;
+
+  RecursivePeerRefreshState(
+      final BytesValue target,
+      final PeerBlacklist peerBlacklist,
+      final BondingAgent bondingAgent,
+      final NeighborFinder neighborFinder) {
+    this.target = target;
+    this.peerBlacklist = peerBlacklist;
+    this.bondingAgent = bondingAgent;
+    this.neighborFinder = neighborFinder;
+    this.anteList = new ArrayList<>();
+    this.outstandingRequestList = new ArrayList<>();
+    this.contactedInCurrentExecution = new ArrayList<>();
+  }
+
+  void kickstartBootstrapPeers(final List<Peer> bootstrapPeers) {
+    for (Peer bootstrapPeer : bootstrapPeers) {
+      final BytesValue peerId = bootstrapPeer.getId();
+      outstandingRequestList.add(new OutstandingRequest(bootstrapPeer));
+      contactedInCurrentExecution.add(peerId);
+      bondingAgent.performBonding(bootstrapPeer, true);
+      neighborFinder.issueFindNodeRequest(bootstrapPeer, target);
+    }
+  }
+
+  /**
+   * This method is intended to be called periodically by the {@link PeerDiscoveryController}, which
+   * will maintain a timer for purposes of effecting expiration of requests outstanding. Requests
+   * once encountered are deemed eligible for eviction if they have not been dispatched before the
+   * next invocation of the method.
+   */
+  public void executeTimeoutEvaluation() {
+    for (int i = 0; i < outstandingRequestList.size(); i++) {
+      if (outstandingRequestList.get(i).getEvaluation()) {
+        final List<DiscoveryPeer> queryCandidates = determineFindNodeCandidates(anteList.size());
+        for (DiscoveryPeer candidate : queryCandidates) {
+          if (!contactedInCurrentExecution.contains(candidate.getId())
+              && !outstandingRequestList.contains(new OutstandingRequest(candidate))) {
+            outstandingRequestList.remove(i);
+            executeFindNodeRequest(candidate);
+          }
+        }
+      }
+      outstandingRequestList.get(i).setEvaluation();
+    }
+  }
+
+  private void executeFindNodeRequest(final DiscoveryPeer peer) {
+    final BytesValue peerId = peer.getId();
+    outstandingRequestList.add(new OutstandingRequest(peer));
+    contactedInCurrentExecution.add(peerId);
+    neighborFinder.issueFindNodeRequest(peer, target);
+  }
+
+  /**
+   * The lookup initiator starts by picking CONCURRENT_REQUEST_LIMIT closest nodes to the target it
+   * knows of. The initiator then issues concurrent FindNode packets to those nodes.
+   */
+  private void initiatePeerRefreshCycle(final List<DiscoveryPeer> peers) {
+    for (DiscoveryPeer peer : peers) {
+      if (!contactedInCurrentExecution.contains(peer.getId())) {
+        executeFindNodeRequest(peer);
+      }
+    }
+  }
+
+  void onNeighboursPacketReceived(final NeighborsPacketData neighboursPacket, final Peer peer) {
+    if (outstandingRequestList.contains(new OutstandingRequest(peer))) {
+      final List<DiscoveryPeer> receivedPeerList = neighboursPacket.getNodes();
+      for (DiscoveryPeer receivedPeer : receivedPeerList) {
+        if (!peerBlacklist.contains(receivedPeer)) {
+          bondingAgent.performBonding(receivedPeer, false);
+          anteList.add(new PeerDistance(receivedPeer, distance(target, receivedPeer.getId())));
+        }
+      }
+      outstandingRequestList.remove(new OutstandingRequest(peer));
+      queryNearestNodes();
+    }
+  }
+
+  private List<DiscoveryPeer> determineFindNodeCandidates(final int threshold) {
+    anteList.sort(
+        (peer1, peer2) -> {
+          if (peer1.getDistance() > peer2.getDistance()) return 1;
+          if (peer1.getDistance() < peer2.getDistance()) return -1;
+          return 0;
+        });
+    return anteList.subList(0, threshold).stream().map(PeerDistance::getPeer).collect(toList());
+  }
+
+  private void queryNearestNodes() {
+    if (outstandingRequestList.isEmpty()) {
+      final List<DiscoveryPeer> queryCandidates =
+          determineFindNodeCandidates(CONCURRENT_REQUEST_LIMIT);
+      initiatePeerRefreshCycle(queryCandidates);
+    }
+  }
+
+  @VisibleForTesting
+  List<OutstandingRequest> getOutstandingRequestList() {
+    return outstandingRequestList;
+  }
+
+  static class PeerDistance {
+    DiscoveryPeer peer;
+    Integer distance;
+
+    PeerDistance(final DiscoveryPeer peer, final Integer distance) {
+      this.peer = peer;
+      this.distance = distance;
+    }
+
+    DiscoveryPeer getPeer() {
+      return peer;
+    }
+
+    Integer getDistance() {
+      return distance;
+    }
+
+    @Override
+    public String toString() {
+      return peer + ": " + distance;
+    }
+  }
+
+  static class OutstandingRequest {
+    boolean evaluation;
+    Peer peer;
+
+    OutstandingRequest(final Peer peer) {
+      this.evaluation = false;
+      this.peer = peer;
+    }
+
+    boolean getEvaluation() {
+      return evaluation;
+    }
+
+    Peer getPeer() {
+      return peer;
+    }
+
+    void setEvaluation() {
+      this.evaluation = true;
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+      final OutstandingRequest that = (OutstandingRequest) o;
+      return Objects.equals(peer.getId(), that.peer.getId());
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(peer.getId());
+    }
+
+    @Override
+    public String toString() {
+      return peer.toString();
+    }
+  }
+
+  public interface NeighborFinder {
+    /**
+     * Sends a FIND_NEIGHBORS message to a {@link DiscoveryPeer}, in search of a target value.
+     *
+     * @param peer the peer to interrogate
+     * @param target the target node ID to find
+     */
+    void issueFindNodeRequest(final Peer peer, final BytesValue target);
+  }
+
+  public interface BondingAgent {
+    /**
+     * Initiates a bonding PING-PONG cycle with a peer.
+     *
+     * @param peer The targeted peer.
+     * @param bootstrap Whether this is a bootstrap interaction.
+     */
+    void performBonding(final Peer peer, final boolean bootstrap);
+  }
+}

ethereum/p2p/src/test/java/tech/pegasys/pantheon/ethereum/p2p/discovery/internal/PeerDiscoveryControllerDistanceCalculatorTest.java

@@ -13,6 +13,7 @@
 package tech.pegasys.pantheon.ethereum.p2p.discovery.internal;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static tech.pegasys.pantheon.ethereum.p2p.discovery.internal.PeerDistanceCalculator.distance;
 
 import tech.pegasys.pantheon.util.bytes.BytesValue;
 
@@ -26,55 +27,55 @@ public class PeerDiscoveryControllerDistanceCalculatorTest {
   public void distanceZero() {
     final byte[] id = new byte[64];
     new Random().nextBytes(id);
-    assertThat(PeerTable.distance(BytesValue.wrap(id), BytesValue.wrap(id))).isEqualTo(0);
+    assertThat(distance(BytesValue.wrap(id), BytesValue.wrap(id))).isEqualTo(0);
   }
 
   @Test
   public void distance1() {
     final BytesValue id1 = BytesValue.fromHexString("0x8f19400000");
     final BytesValue id2 = BytesValue.fromHexString("0x8f19400001");
-    assertThat(PeerTable.distance(id1, id2)).isEqualTo(1);
+    assertThat(distance(id1, id2)).isEqualTo(1);
   }
 
   @Test
   public void distance2() {
     final BytesValue id1 = BytesValue.fromHexString("0x8f19400000");
     final BytesValue id2 = BytesValue.fromHexString("0x8f19400002");
-    assertThat(PeerTable.distance(id1, id2)).isEqualTo(2);
+    assertThat(distance(id1, id2)).isEqualTo(2);
   }
 
   @Test
   public void distance3() {
     final BytesValue id1 = BytesValue.fromHexString("0x8f19400000");
     final BytesValue id2 = BytesValue.fromHexString("0x8f19400004");
-    assertThat(PeerTable.distance(id1, id2)).isEqualTo(3);
+    assertThat(distance(id1, id2)).isEqualTo(3);
   }
 
   @Test
   public void distance9() {
     final BytesValue id1 = BytesValue.fromHexString("0x8f19400100");
     final BytesValue id2 = BytesValue.fromHexString("0x8f19400000");
-    assertThat(PeerTable.distance(id1, id2)).isEqualTo(9);
+    assertThat(distance(id1, id2)).isEqualTo(9);
   }
 
   @Test
   public void distance40() {
     final BytesValue id1 = BytesValue.fromHexString("0x8f19400000");
     final BytesValue id2 = BytesValue.fromHexString("0x0f19400000");
-    assertThat(PeerTable.distance(id1, id2)).isEqualTo(40);
+    assertThat(distance(id1, id2)).isEqualTo(40);
   }
 
   @Test(expected = AssertionError.class)
   public void distance40_differentLengths() {
     final BytesValue id1 = BytesValue.fromHexString("0x8f19400000");
     final BytesValue id2 = BytesValue.fromHexString("0x0f1940000099");
-    assertThat(PeerTable.distance(id1, id2)).isEqualTo(40);
+    assertThat(distance(id1, id2)).isEqualTo(40);
   }
 
   @Test
   public void distanceZero_emptyArrays() {
     final BytesValue id1 = BytesValue.EMPTY;
     final BytesValue id2 = BytesValue.EMPTY;
-    assertThat(PeerTable.distance(id1, id2)).isEqualTo(0);
+    assertThat(distance(id1, id2)).isEqualTo(0);
   }
 }