From c00378909453ee58080aa4d30ba1f4794cff581b Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 20 Dec 2023 08:47:37 +0000 Subject: [PATCH] feat!: use single DHT only by default (#2322) We have a weird dual-DHT setup whereby we have a DHT for public addresses (amino, `/ipfs/kad/1.0.0`) and a DHT for private addresess (`/ipfs/lan/kad/1.0.0`). This is an artefact of the previous libp2p setup whereby there could only be a single DHT implementation at once. Now we have services we can configure an amino DHT service and if desired an additional lan-only DHT, or the user can just use amino and not pay the overhead of running an extra DHT. Most content is resolved via the public DHT so running the lan-only DHT gives the user no benefit. BREAKING CHANGE: the `kadDHT` function returns a single DHT - see the readme for how to configure amino/lan as before --- packages/integration-tests/test/interop.ts | 43 +- packages/kad-dht/README.md | 78 ++ packages/kad-dht/package.json | 2 + packages/kad-dht/src/constants.ts | 18 +- .../kad-dht/src/content-fetching/index.ts | 35 +- packages/kad-dht/src/content-routing/index.ts | 47 +- packages/kad-dht/src/dual-kad-dht.ts | 384 --------- packages/kad-dht/src/index.ts | 180 +++- packages/kad-dht/src/kad-dht.ts | 281 ++++-- packages/kad-dht/src/message/dht.proto | 64 +- packages/kad-dht/src/message/dht.ts | 293 ++++--- packages/kad-dht/src/message/index.ts | 117 --- packages/kad-dht/src/message/utils.ts | 25 + packages/kad-dht/src/network.ts | 66 +- packages/kad-dht/src/peer-routing/index.ts | 99 ++- packages/kad-dht/src/providers.ts | 7 + packages/kad-dht/src/query-self.ts | 6 +- packages/kad-dht/src/query/events.ts | 28 +- packages/kad-dht/src/query/manager.ts | 24 +- packages/kad-dht/src/query/query-path.ts | 6 +- packages/kad-dht/src/routing-table/index.ts | 150 ++-- packages/kad-dht/src/routing-table/refresh.ts | 8 +- .../kad-dht/src/rpc/handlers/add-provider.ts | 17 +- .../kad-dht/src/rpc/handlers/find-node.ts | 45 +- .../kad-dht/src/rpc/handlers/get-providers.ts | 53 +- .../kad-dht/src/rpc/handlers/get-value.ts | 25 +- packages/kad-dht/src/rpc/handlers/ping.ts | 10 +- .../kad-dht/src/rpc/handlers/put-value.ts | 20 +- packages/kad-dht/src/rpc/index.ts | 27 +- packages/kad-dht/src/topology-listener.ts | 6 +- packages/kad-dht/src/utils.ts | 43 +- .../kad-dht/test/enable-server-mode.spec.ts | 8 +- .../kad-dht/test/fixtures/match-peer-id.ts | 6 + .../generate-peers/generate-peers.node.ts | 4 +- packages/kad-dht/test/kad-dht.spec.ts | 137 +-- packages/kad-dht/test/kad-utils.spec.ts | 4 +- packages/kad-dht/test/libp2p-routing.spec.ts | 801 ++++++++++++++++++ packages/kad-dht/test/message.node.ts | 16 +- packages/kad-dht/test/message.spec.ts | 62 +- packages/kad-dht/test/multiple-nodes.spec.ts | 4 +- packages/kad-dht/test/network.spec.ts | 33 +- packages/kad-dht/test/query-self.spec.ts | 3 +- packages/kad-dht/test/query.spec.ts | 29 +- packages/kad-dht/test/routing-table.spec.ts | 73 +- .../test/rpc/handlers/add-provider.spec.ts | 38 +- .../test/rpc/handlers/find-node.spec.ts | 91 +- .../test/rpc/handlers/get-providers.spec.ts | 31 +- .../test/rpc/handlers/get-value.spec.ts | 64 +- .../kad-dht/test/rpc/handlers/ping.spec.ts | 13 +- .../test/rpc/handlers/put-value.spec.ts | 23 +- packages/kad-dht/test/rpc/index.node.ts | 17 +- packages/kad-dht/test/utils/test-dht.ts | 77 +- 52 files changed, 2318 insertions(+), 1423 deletions(-) delete mode 100644 packages/kad-dht/src/dual-kad-dht.ts delete mode 100644 packages/kad-dht/src/message/index.ts create mode 100644 packages/kad-dht/src/message/utils.ts create mode 100644 packages/kad-dht/test/fixtures/match-peer-id.ts create mode 100644 packages/kad-dht/test/libp2p-routing.spec.ts diff --git a/packages/integration-tests/test/interop.ts b/packages/integration-tests/test/interop.ts index ce2802eee7..7de8aefe41 100644 --- a/packages/integration-tests/test/interop.ts +++ b/packages/integration-tests/test/interop.ts @@ -8,9 +8,8 @@ import { createClient } from '@libp2p/daemon-client' import { createServer } from '@libp2p/daemon-server' import { floodsub } from '@libp2p/floodsub' import { identify } from '@libp2p/identify' -import { contentRoutingSymbol, peerDiscoverySymbol, peerRoutingSymbol } from '@libp2p/interface' import { interopTests } from '@libp2p/interop' -import { kadDHT } from '@libp2p/kad-dht' +import { kadDHT, passthroughMapper } from '@libp2p/kad-dht' import { logger } from '@libp2p/logger' import { mplex } from '@libp2p/mplex' import { peerIdFromKeys } from '@libp2p/peer-id' @@ -155,41 +154,11 @@ async function createJsPeer (options: SpawnOptions): Promise { } if (options.dht === true) { - services.dht = (components: any) => { - const dht: any = kadDHT({ - clientMode: false - })(components) - - // go-libp2p-daemon only has the older single-table DHT instead of the dual - // lan/wan version found in recent go-ipfs versions. unfortunately it's been - // abandoned so here we simulate the older config with the js implementation - const lan: any = dht.lan - - const protocol = '/ipfs/kad/1.0.0' - lan.protocol = protocol - lan.network.protocol = protocol - lan.topologyListener.protocol = protocol - - Object.defineProperties(lan, { - [contentRoutingSymbol]: { - get () { - return dht[contentRoutingSymbol] - } - }, - [peerRoutingSymbol]: { - get () { - return dht[peerRoutingSymbol] - } - }, - [peerDiscoverySymbol]: { - get () { - return dht[peerDiscoverySymbol] - } - } - }) - - return lan - } + services.dht = kadDHT({ + protocol: '/ipfs/kad/1.0.0', + peerInfoMapper: passthroughMapper, + clientMode: false + }) } const node: any = await createLibp2p({ diff --git a/packages/kad-dht/README.md b/packages/kad-dht/README.md index de834ebfe5..5f17270af5 100644 --- a/packages/kad-dht/README.md +++ b/packages/kad-dht/README.md @@ -5,6 +5,84 @@ > JavaScript implementation of the Kad-DHT for libp2p +# About + +This module implements the [libp2p Kademlia spec](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) in TypeScript. + +The Kademlia DHT allow for several operations such as finding peers, searching for providers of DHT records, etc. + +## Example - Using with libp2p + +```TypeScript +import { kadDHT } from '@libp2p/kad-dht' +import { createLibp2p } from 'libp2p' +import { peerIdFromString } from '@libp2p/peer-id' + +const node = await createLibp2p({ + services: { + dht: kadDHT() + } +}) + +const peerId = peerIdFromString('QmFoo') +const peerInfo = await libp2p.peerRouting.findPeer(peerId) + +console.info(peerInfo) // peer id, multiaddrs +``` + +## Example - Connecting to the IPFS Amino DHT + +The [Amino DHT](https://blog.ipfs.tech/2023-09-amino-refactoring/) is a public-good DHT used by IPFS to fetch content, find peers, etc. + +If you are trying to access content on the public internet, this is the implementation you want. + +```TypeScript +import { kadDHT, removePrivateAddressesMapper } from '@libp2p/kad-dht' +import { createLibp2p } from 'libp2p' +import { peerIdFromString } from '@libp2p/peer-id' + +const node = await createLibp2p({ + services: { + aminoDHT: kadDHT({ + protocol: '/ipfs/kad/1.0.0', + addressFilter: removePrivateAddressesMapper + }) + } +}) + +const peerId = peerIdFromString('QmFoo') +const peerInfo = await libp2p.peerRouting.findPeer(peerId) + +console.info(peerInfo) // peer id, multiaddrs +``` + +## Example - Connecting to a LAN-only DHT + +This DHT only works with privately dialable peers. + +This is for use when peers are on the local area network. + +```TypeScript +import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht' +import { createLibp2p } from 'libp2p' +import { peerIdFromString } from '@libp2p/peer-id' + +const node = await createLibp2p({ + services: { + lanDHT: kadDHT({ + protocol: '/ipfs/lan/kad/1.0.0', + addressFilter: removePublicAddressesMapper, + clientMode: false + }) + } +}) + +const peerId = peerIdFromString('QmFoo') +const peerInfo = await libp2p.peerRouting.findPeer(peerId) + +console.info(peerInfo) // peer id, multiaddrs +``` + # Install ```console diff --git a/packages/kad-dht/package.json b/packages/kad-dht/package.json index f9619ac52e..5c35e20d1d 100644 --- a/packages/kad-dht/package.json +++ b/packages/kad-dht/package.json @@ -61,6 +61,7 @@ "@libp2p/interface-internal": "^1.0.3", "@libp2p/peer-collections": "^5.1.1", "@libp2p/peer-id": "^4.0.2", + "@libp2p/utils": "^5.0.3", "@multiformats/multiaddr": "^12.1.10", "@types/sinon": "^17.0.0", "any-signal": "^4.1.1", @@ -104,6 +105,7 @@ "execa": "^8.0.1", "it-filter": "^3.0.1", "it-last": "^3.0.3", + "it-pair": "^2.0.6", "lodash.random": "^3.2.0", "lodash.range": "^3.2.0", "p-retry": "^6.1.0", diff --git a/packages/kad-dht/src/constants.ts b/packages/kad-dht/src/constants.ts index 12edd7f512..54d4de61b6 100644 --- a/packages/kad-dht/src/constants.ts +++ b/packages/kad-dht/src/constants.ts @@ -11,11 +11,7 @@ export const hour = 60 * minute export const MAX_RECORD_AGE = 36 * hour -export const LAN_PREFIX = '/lan' - -export const PROTOCOL_PREFIX = '/ipfs' - -export const PROTOCOL_DHT = '/kad/1.0.0' +export const PROTOCOL = '/ipfs/kad/1.0.0' export const RECORD_KEY_PREFIX = '/dht/record' @@ -39,19 +35,19 @@ export const K = 20 export const ALPHA = 3 // How often we look for our closest DHT neighbours -export const QUERY_SELF_INTERVAL = Number(5 * minute) +export const QUERY_SELF_INTERVAL = 5 * minute // How often we look for the first set of our closest DHT neighbours -export const QUERY_SELF_INITIAL_INTERVAL = Number(Number(second)) +export const QUERY_SELF_INITIAL_INTERVAL = second // How long to look for our closest DHT neighbours for -export const QUERY_SELF_TIMEOUT = Number(5 * second) +export const QUERY_SELF_TIMEOUT = 5 * second // How often we try to find new peers -export const TABLE_REFRESH_INTERVAL = Number(5 * minute) +export const TABLE_REFRESH_INTERVAL = 5 * minute // How how long to look for new peers for -export const TABLE_REFRESH_QUERY_TIMEOUT = Number(30 * second) +export const TABLE_REFRESH_QUERY_TIMEOUT = 30 * second // When a timeout is not specified, run a query for this long -export const DEFAULT_QUERY_TIMEOUT = Number(30 * second) +export const DEFAULT_QUERY_TIMEOUT = 30 * second diff --git a/packages/kad-dht/src/content-fetching/index.ts b/packages/kad-dht/src/content-fetching/index.ts index 32c2d87ff7..80a3c9ae6c 100644 --- a/packages/kad-dht/src/content-fetching/index.ts +++ b/packages/kad-dht/src/content-fetching/index.ts @@ -6,7 +6,7 @@ import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import { ALPHA } from '../constants.js' -import { Message, MESSAGE_TYPE } from '../message/index.js' +import { MessageType } from '../message/dht.js' import { valueEvent, queryErrorEvent @@ -15,12 +15,13 @@ import { Libp2pRecord } from '../record/index.js' import { bestRecord } from '../record/selectors.js' import { verifyRecord } from '../record/validators.js' import { createPutRecord, bufferToRecordKey } from '../utils.js' -import type { KadDHTComponents, Validators, Selectors, ValueEvent, QueryOptions, QueryEvent } from '../index.js' +import type { KadDHTComponents, Validators, Selectors, ValueEvent, QueryEvent } from '../index.js' +import type { Message } from '../message/dht.js' import type { Network } from '../network.js' import type { PeerRouting } from '../peer-routing/index.js' import type { QueryManager } from '../query/manager.js' import type { QueryFunc } from '../query/types.js' -import type { AbortOptions, Logger } from '@libp2p/interface' +import type { Logger, RoutingOptions } from '@libp2p/interface' export interface ContentFetchingInit { validators: Validators @@ -28,7 +29,7 @@ export interface ContentFetchingInit { peerRouting: PeerRouting queryManager: QueryManager network: Network - lan: boolean + logPrefix: string } export class ContentFetching { @@ -41,10 +42,10 @@ export class ContentFetching { private readonly network: Network constructor (components: KadDHTComponents, init: ContentFetchingInit) { - const { validators, selectors, peerRouting, queryManager, network, lan } = init + const { validators, selectors, peerRouting, queryManager, network, logPrefix } = init this.components = components - this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:content-fetching`) + this.log = components.logger.forComponent(`${logPrefix}:content-fetching`) this.validators = validators this.selectors = selectors this.peerRouting = peerRouting @@ -81,7 +82,7 @@ export class ContentFetching { /** * Send the best record found to any peers that have an out of date record */ - async * sendCorrectionRecord (key: Uint8Array, vals: ValueEvent[], best: Uint8Array, options: AbortOptions = {}): AsyncGenerator { + async * sendCorrectionRecord (key: Uint8Array, vals: ValueEvent[], best: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { this.log('sendCorrection for %b', key) const fixupRec = createPutRecord(key, best) @@ -107,8 +108,11 @@ export class ContentFetching { // send correction let sentCorrection = false - const request = new Message(MESSAGE_TYPE.PUT_VALUE, key, 0) - request.record = Libp2pRecord.deserialize(fixupRec) + const request: Partial = { + type: MessageType.PUT_VALUE, + key, + record: fixupRec + } for await (const event of this.network.sendRequest(from, request, options)) { if (event.name === 'PEER_RESPONSE' && (event.record != null) && uint8ArrayEquals(event.record.value, Libp2pRecord.deserialize(fixupRec).value)) { @@ -129,7 +133,7 @@ export class ContentFetching { /** * Store the given key/value pair in the DHT */ - async * put (key: Uint8Array, value: Uint8Array, options: AbortOptions = {}): AsyncGenerator { + async * put (key: Uint8Array, value: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { this.log('put key %b value %b', key, value) // create record in the dht format @@ -151,8 +155,11 @@ export class ContentFetching { const events = [] - const msg = new Message(MESSAGE_TYPE.PUT_VALUE, key, 0) - msg.record = Libp2pRecord.deserialize(record) + const msg: Partial = { + type: MessageType.PUT_VALUE, + key, + record + } this.log('send put to %p', event.peer.id) for await (const putEvent of this.network.sendRequest(event.peer.id, msg, options)) { @@ -185,7 +192,7 @@ export class ContentFetching { /** * Get the value to the given key */ - async * get (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator { + async * get (key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { this.log('get %b', key) const vals: ValueEvent[] = [] @@ -229,7 +236,7 @@ export class ContentFetching { /** * Get the `n` values to the given key without sorting */ - async * getMany (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator { + async * getMany (key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { this.log('getMany values for %b', key) try { diff --git a/packages/kad-dht/src/content-routing/index.ts b/packages/kad-dht/src/content-routing/index.ts index 22e433b242..2fe4e03964 100644 --- a/packages/kad-dht/src/content-routing/index.ts +++ b/packages/kad-dht/src/content-routing/index.ts @@ -1,21 +1,24 @@ +import { PeerSet } from '@libp2p/peer-collections' import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { ALPHA } from '../constants.js' -import { Message, MESSAGE_TYPE } from '../message/index.js' +import { MessageType } from '../message/dht.js' +import { toPbPeerInfo } from '../message/utils.js' import { queryErrorEvent, peerResponseEvent, providerEvent } from '../query/events.js' -import type { KadDHTComponents, PeerResponseEvent, ProviderEvent, QueryEvent, QueryOptions } from '../index.js' +import type { KadDHTComponents, PeerResponseEvent, ProviderEvent, QueryEvent } from '../index.js' +import type { Message } from '../message/dht.js' import type { Network } from '../network.js' import type { PeerRouting } from '../peer-routing/index.js' import type { Providers } from '../providers.js' import type { QueryManager } from '../query/manager.js' import type { QueryFunc } from '../query/types.js' import type { RoutingTable } from '../routing-table/index.js' -import type { Logger, PeerInfo } from '@libp2p/interface' +import type { Logger, PeerInfo, RoutingOptions } from '@libp2p/interface' import type { Multiaddr } from '@multiformats/multiaddr' import type { CID } from 'multiformats/cid' @@ -25,7 +28,7 @@ export interface ContentRoutingInit { queryManager: QueryManager routingTable: RoutingTable providers: Providers - lan: boolean + logPrefix: string } export class ContentRouting { @@ -38,10 +41,10 @@ export class ContentRouting { private readonly providers: Providers constructor (components: KadDHTComponents, init: ContentRoutingInit) { - const { network, peerRouting, queryManager, routingTable, providers, lan } = init + const { network, peerRouting, queryManager, routingTable, providers, logPrefix } = init this.components = components - this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:content-routing`) + this.log = components.logger.forComponent(`${logPrefix}:content-routing`) this.network = network this.peerRouting = peerRouting this.queryManager = queryManager @@ -53,17 +56,22 @@ export class ContentRouting { * Announce to the network that we can provide the value for a given key and * are contactable on the given multiaddrs */ - async * provide (key: CID, multiaddrs: Multiaddr[], options: QueryOptions = {}): AsyncGenerator { + async * provide (key: CID, multiaddrs: Multiaddr[], options: RoutingOptions = {}): AsyncGenerator { this.log('provide %s', key) // Add peer as provider await this.providers.addProvider(key, this.components.peerId) - const msg = new Message(MESSAGE_TYPE.ADD_PROVIDER, key.multihash.bytes, 0) - msg.providerPeers = [{ - id: this.components.peerId, - multiaddrs - }] + const msg: Partial = { + type: MessageType.ADD_PROVIDER, + key: key.multihash.bytes, + providers: [ + toPbPeerInfo({ + id: this.components.peerId, + multiaddrs + }) + ] + } let sent = 0 @@ -118,7 +126,7 @@ export class ContentRouting { /** * Search the dht for up to `K` providers of the given CID. */ - async * findProviders (key: CID, options: QueryOptions): AsyncGenerator { + async * findProviders (key: CID, options: RoutingOptions): AsyncGenerator { const toFind = this.routingTable.kBucketSize const target = key.multihash.bytes const self = this // eslint-disable-line @typescript-eslint/no-this-alias @@ -148,7 +156,7 @@ export class ContentRouting { } } - yield peerResponseEvent({ from: this.components.peerId, messageType: MESSAGE_TYPE.GET_PROVIDERS, providers }, options) + yield peerResponseEvent({ from: this.components.peerId, messageType: MessageType.GET_PROVIDERS, providers }, options) yield providerEvent({ from: this.components.peerId, providers }, options) } @@ -161,7 +169,10 @@ export class ContentRouting { * The query function to use on this particular disjoint path */ const findProvidersQuery: QueryFunc = async function * ({ peer, signal }) { - const request = new Message(MESSAGE_TYPE.GET_PROVIDERS, target, 0) + const request = { + type: MessageType.GET_PROVIDERS, + key: target + } yield * self.network.sendRequest(peer, request, { ...options, @@ -169,7 +180,7 @@ export class ContentRouting { }) } - const providers = new Set(provs.map(p => p.toString())) + const providers = new PeerSet(provs) for await (const event of this.queryManager.run(target, findProvidersQuery, options)) { yield event @@ -180,11 +191,11 @@ export class ContentRouting { const newProviders = [] for (const peer of event.providers) { - if (providers.has(peer.id.toString())) { + if (providers.has(peer.id)) { continue } - providers.add(peer.id.toString()) + providers.add(peer.id) newProviders.push(peer) } diff --git a/packages/kad-dht/src/dual-kad-dht.ts b/packages/kad-dht/src/dual-kad-dht.ts deleted file mode 100644 index 45a2f0982e..0000000000 --- a/packages/kad-dht/src/dual-kad-dht.ts +++ /dev/null @@ -1,384 +0,0 @@ -import { contentRoutingSymbol, peerDiscoverySymbol, peerRoutingSymbol, CodeError, TypedEventEmitter, CustomEvent } from '@libp2p/interface' -import drain from 'it-drain' -import merge from 'it-merge' -import isPrivate from 'private-ip' -import { DefaultKadDHT } from './kad-dht.js' -import { queryErrorEvent } from './query/events.js' -import type { DualKadDHT, KadDHT, KadDHTComponents, KadDHTInit, QueryEvent, QueryOptions } from './index.js' -import type { PeerDiscovery, PeerDiscoveryEvents, PeerRouting, ContentRouting, Logger, PeerId, PeerInfo } from '@libp2p/interface' -import type { Multiaddr } from '@multiformats/multiaddr' -import type { CID } from 'multiformats/cid' - -/** - * Wrapper class to convert events into returned values - */ -class DHTContentRouting implements ContentRouting { - private readonly dht: KadDHT - - constructor (dht: KadDHT) { - this.dht = dht - } - - async provide (cid: CID, options: QueryOptions = {}): Promise { - await drain(this.dht.provide(cid, options)) - } - - async * findProviders (cid: CID, options: QueryOptions = {}): AsyncGenerator { - for await (const event of this.dht.findProviders(cid, options)) { - if (event.name === 'PROVIDER') { - yield * event.providers - } - } - } - - async put (key: Uint8Array, value: Uint8Array, options?: QueryOptions): Promise { - await drain(this.dht.put(key, value, options)) - } - - async get (key: Uint8Array, options?: QueryOptions): Promise { - for await (const event of this.dht.get(key, options)) { - if (event.name === 'VALUE') { - return event.value - } - } - - throw new CodeError('Not found', 'ERR_NOT_FOUND') - } -} - -/** - * Wrapper class to convert events into returned values - */ -class DHTPeerRouting implements PeerRouting { - private readonly dht: KadDHT - - constructor (dht: KadDHT) { - this.dht = dht - } - - async findPeer (peerId: PeerId, options: QueryOptions = {}): Promise { - for await (const event of this.dht.findPeer(peerId, options)) { - if (event.name === 'FINAL_PEER') { - return event.peer - } - } - - throw new CodeError('Not found', 'ERR_NOT_FOUND') - } - - async * getClosestPeers (key: Uint8Array, options: QueryOptions = {}): AsyncIterable { - for await (const event of this.dht.getClosestPeers(key, options)) { - if (event.name === 'FINAL_PEER') { - yield event.peer - } - } - } -} - -// see https://github.com/multiformats/multiaddr/blob/master/protocols.csv -const P2P_CIRCUIT_CODE = 290 -const DNS4_CODE = 54 -const DNS6_CODE = 55 -const DNSADDR_CODE = 56 -const IP4_CODE = 4 -const IP6_CODE = 41 - -function multiaddrIsPublic (multiaddr: Multiaddr): boolean { - const tuples = multiaddr.stringTuples() - - // p2p-circuit should not enable server mode - for (const tuple of tuples) { - if (tuple[0] === P2P_CIRCUIT_CODE) { - return false - } - } - - // dns4 or dns6 or dnsaddr - if (tuples[0][0] === DNS4_CODE || tuples[0][0] === DNS6_CODE || tuples[0][0] === DNSADDR_CODE) { - return true - } - - // ip4 or ip6 - if (tuples[0][0] === IP4_CODE || tuples[0][0] === IP6_CODE) { - const result = isPrivate(`${tuples[0][1]}`) - const isPublic = result == null || !result - - return isPublic - } - - return false -} - -/** - * A DHT implementation modelled after Kademlia with S/Kademlia modifications. - * Original implementation in go: https://github.com/libp2p/go-libp2p-kad-dht. - */ -export class DefaultDualKadDHT extends TypedEventEmitter implements DualKadDHT, PeerDiscovery { - public readonly wan: DefaultKadDHT - public readonly lan: DefaultKadDHT - public readonly components: KadDHTComponents - private readonly contentRouting: ContentRouting - private readonly peerRouting: PeerRouting - private readonly log: Logger - - constructor (components: KadDHTComponents, init: KadDHTInit = {}) { - super() - - this.components = components - this.log = components.logger.forComponent('libp2p:kad-dht') - - this.wan = new DefaultKadDHT(components, { - protocolPrefix: '/ipfs', - ...init, - lan: false - }) - this.lan = new DefaultKadDHT(components, { - protocolPrefix: '/ipfs', - ...init, - clientMode: false, - lan: true - }) - - this.contentRouting = new DHTContentRouting(this) - this.peerRouting = new DHTPeerRouting(this) - - // handle peers being discovered during processing of DHT messages - this.wan.addEventListener('peer', (evt) => { - this.dispatchEvent(new CustomEvent('peer', { - detail: evt.detail - })) - }) - this.lan.addEventListener('peer', (evt) => { - this.dispatchEvent(new CustomEvent('peer', { - detail: evt.detail - })) - }) - - // if client mode has not been explicitly specified, auto-switch to server - // mode when the node's peer data is updated with publicly dialable addresses - if (init.clientMode == null) { - components.events.addEventListener('self:peer:update', (evt) => { - this.log('received update of self-peer info') - const hasPublicAddress = evt.detail.peer.addresses - .some(({ multiaddr }) => multiaddrIsPublic(multiaddr)) - - this.getMode() - .then(async mode => { - if (hasPublicAddress && mode === 'client') { - await this.setMode('server') - } else if (mode === 'server' && !hasPublicAddress) { - await this.setMode('client') - } - }) - .catch(err => { - this.log.error('error setting dht server mode', err) - }) - }) - } - } - - readonly [Symbol.toStringTag] = '@libp2p/dual-kad-dht' - - get [contentRoutingSymbol] (): ContentRouting { - return this.contentRouting - } - - get [peerRoutingSymbol] (): PeerRouting { - return this.peerRouting - } - - get [peerDiscoverySymbol] (): PeerDiscovery { - return this - } - - /** - * Is this DHT running. - */ - isStarted (): boolean { - return this.wan.isStarted() && this.lan.isStarted() - } - - /** - * If 'server' this node will respond to DHT queries, if 'client' this node will not - */ - async getMode (): Promise<'client' | 'server'> { - return this.wan.getMode() - } - - /** - * If 'server' this node will respond to DHT queries, if 'client' this node will not - */ - async setMode (mode: 'client' | 'server'): Promise { - await this.wan.setMode(mode) - } - - /** - * Start listening to incoming connections. - */ - async start (): Promise { - await Promise.all([ - this.lan.start(), - this.wan.start() - ]) - } - - /** - * Stop accepting incoming connections and sending outgoing - * messages. - */ - async stop (): Promise { - await Promise.all([ - this.lan.stop(), - this.wan.stop() - ]) - } - - /** - * Store the given key/value pair in the DHT - */ - async * put (key: Uint8Array, value: Uint8Array, options: QueryOptions = {}): AsyncGenerator { - for await (const event of merge( - this.lan.put(key, value, options), - this.wan.put(key, value, options) - )) { - yield event - } - } - - /** - * Get the value that corresponds to the passed key - */ - async * get (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator { - let queriedPeers = false - let foundValue = false - - for await (const event of merge( - this.lan.get(key, options), - this.wan.get(key, options) - )) { - yield event - - if (event.name === 'DIAL_PEER') { - queriedPeers = true - } - - if (event.name === 'VALUE') { - queriedPeers = true - - if (event.value != null) { - foundValue = true - } - } - - if (event.name === 'SEND_QUERY') { - queriedPeers = true - } - } - - if (!queriedPeers) { - throw new CodeError('No peers found in routing table!', 'ERR_NO_PEERS_IN_ROUTING_TABLE') - } - - if (!foundValue) { - yield queryErrorEvent({ - from: this.components.peerId, - error: new CodeError('Not found', 'ERR_NOT_FOUND') - }, options) - } - } - - // ----------- Content Routing - - /** - * Announce to the network that we can provide given key's value - */ - async * provide (key: CID, options: QueryOptions = {}): AsyncGenerator { - let sent = 0 - let success = 0 - const errors = [] - - const dhts = [this.lan] - - // only run provide on the wan if we are in server mode - if ((await this.wan.getMode()) === 'server') { - dhts.push(this.wan) - } - - for await (const event of merge(...dhts.map(dht => dht.provide(key, options)))) { - yield event - - if (event.name === 'SEND_QUERY') { - sent++ - } - - if (event.name === 'QUERY_ERROR') { - errors.push(event.error) - } - - if (event.name === 'PEER_RESPONSE' && event.messageName === 'ADD_PROVIDER') { - this.log('sent provider record for %s to %p', key, event.from) - success++ - } - } - - if (success === 0) { - if (errors.length > 0) { - // if all sends failed, throw an error to inform the caller - throw new CodeError(`Failed to provide to ${errors.length} of ${sent} peers`, 'ERR_PROVIDES_FAILED', { errors }) - } - - throw new CodeError('Failed to provide - no peers found', 'ERR_PROVIDES_FAILED') - } - } - - /** - * Search the dht for up to `K` providers of the given CID - */ - async * findProviders (key: CID, options: QueryOptions = {}): AsyncGenerator { - yield * merge( - this.lan.findProviders(key, options), - this.wan.findProviders(key, options) - ) - } - - // ----------- Peer Routing ----------- - - /** - * Search for a peer with the given ID - */ - async * findPeer (id: PeerId, options: QueryOptions = {}): AsyncGenerator { - let queriedPeers = false - - for await (const event of merge( - this.lan.findPeer(id, options), - this.wan.findPeer(id, options) - )) { - yield event - - if (event.name === 'SEND_QUERY' || event.name === 'FINAL_PEER') { - queriedPeers = true - } - } - - if (!queriedPeers) { - throw new CodeError('Peer lookup failed', 'ERR_LOOKUP_FAILED') - } - } - - /** - * Kademlia 'node lookup' operation - */ - async * getClosestPeers (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator { - yield * merge( - this.lan.getClosestPeers(key, options), - this.wan.getClosestPeers(key, options) - ) - } - - async refreshRoutingTable (): Promise { - await Promise.all([ - this.lan.refreshRoutingTable(), - this.wan.refreshRoutingTable() - ]) - } -} diff --git a/packages/kad-dht/src/index.ts b/packages/kad-dht/src/index.ts index d2c5c4fe72..59edfd9870 100644 --- a/packages/kad-dht/src/index.ts +++ b/packages/kad-dht/src/index.ts @@ -1,12 +1,94 @@ -import { DefaultDualKadDHT } from './dual-kad-dht.js' +/** + * @packageDocumentation + * + * This module implements the [libp2p Kademlia spec](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) in TypeScript. + * + * The Kademlia DHT allow for several operations such as finding peers, searching for providers of DHT records, etc. + * + * @example Using with libp2p + * + * ```TypeScript + * import { kadDHT } from '@libp2p/kad-dht' + * import { createLibp2p } from 'libp2p' + * import { peerIdFromString } from '@libp2p/peer-id' + * + * const node = await createLibp2p({ + * services: { + * dht: kadDHT() + * } + * }) + * + * const peerId = peerIdFromString('QmFoo') + * const peerInfo = await libp2p.peerRouting.findPeer(peerId) + * + * console.info(peerInfo) // peer id, multiaddrs + * ``` + * + * @example Connecting to the IPFS Amino DHT + * + * The [Amino DHT](https://blog.ipfs.tech/2023-09-amino-refactoring/) is a public-good DHT used by IPFS to fetch content, find peers, etc. + * + * If you are trying to access content on the public internet, this is the implementation you want. + * + * ```TypeScript + * import { kadDHT, removePrivateAddressesMapper } from '@libp2p/kad-dht' + * import { createLibp2p } from 'libp2p' + * import { peerIdFromString } from '@libp2p/peer-id' + * + * const node = await createLibp2p({ + * services: { + * aminoDHT: kadDHT({ + * protocol: '/ipfs/kad/1.0.0', + * addressFilter: removePrivateAddressesMapper + * }) + * } + * }) + * + * const peerId = peerIdFromString('QmFoo') + * const peerInfo = await libp2p.peerRouting.findPeer(peerId) + * + * console.info(peerInfo) // peer id, multiaddrs + * ``` + * + * @example Connecting to a LAN-only DHT + * + * This DHT only works with privately dialable peers. + * + * This is for use when peers are on the local area network. + * + * ```TypeScript + * import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht' + * import { createLibp2p } from 'libp2p' + * import { peerIdFromString } from '@libp2p/peer-id' + * + * const node = await createLibp2p({ + * services: { + * lanDHT: kadDHT({ + * protocol: '/ipfs/lan/kad/1.0.0', + * addressFilter: removePublicAddressesMapper, + * clientMode: false + * }) + * } + * }) + * + * const peerId = peerIdFromString('QmFoo') + * const peerInfo = await libp2p.peerRouting.findPeer(peerId) + * + * console.info(peerInfo) // peer id, multiaddrs + * ``` + */ + +import { KadDHT as KadDHTClass } from './kad-dht.js' +import { removePrivateAddressesMapper, removePublicAddressesMapper, passthroughMapper } from './utils.js' import type { ProvidersInit } from './providers.js' -import type { Libp2pEvents, AbortOptions, ComponentLogger, TypedEventTarget, Metrics, PeerId, PeerInfo, PeerStore } from '@libp2p/interface' +import type { Libp2pEvents, ComponentLogger, TypedEventTarget, Metrics, PeerId, PeerInfo, PeerStore, RoutingOptions } from '@libp2p/interface' import type { AddressManager, ConnectionManager, Registrar } from '@libp2p/interface-internal' import type { Datastore } from 'interface-datastore' import type { CID } from 'multiformats/cid' -import type { ProgressOptions, ProgressEvent } from 'progress-events' +import type { ProgressEvent } from 'progress-events' export { Libp2pRecord as Record } from './record/index.js' +export { removePrivateAddressesMapper, removePublicAddressesMapper, passthroughMapper } /** * The types of events emitted during DHT queries @@ -52,10 +134,6 @@ export type DHTProgressEvents = ProgressEvent<'kad-dht:query:add-peer', AddPeerEvent> | ProgressEvent<'kad-dht:query:dial-peer', DialPeerEvent> -export interface QueryOptions extends AbortOptions, ProgressOptions { - queryFuncTimeout?: number -} - /** * Emitted when sending queries to remote peers */ @@ -146,44 +224,49 @@ export interface RoutingTable { size: number } +export interface PeerInfoMapper { + (peer: PeerInfo): PeerInfo +} + export interface KadDHT { /** * Get a value from the DHT, the final ValueEvent will be the best value */ - get(key: Uint8Array, options?: QueryOptions): AsyncIterable + get(key: Uint8Array, options?: RoutingOptions): AsyncIterable /** * Find providers of a given CID */ - findProviders(key: CID, options?: QueryOptions): AsyncIterable + findProviders(key: CID, options?: RoutingOptions): AsyncIterable /** * Find a peer on the DHT */ - findPeer(id: PeerId, options?: QueryOptions): AsyncIterable + findPeer(id: PeerId, options?: RoutingOptions): AsyncIterable /** * Find the closest peers to the passed key */ - getClosestPeers(key: Uint8Array, options?: QueryOptions): AsyncIterable + getClosestPeers(key: Uint8Array, options?: RoutingOptions): AsyncIterable /** * Store provider records for the passed CID on the DHT pointing to us */ - provide(key: CID, options?: QueryOptions): AsyncIterable + provide(key: CID, options?: RoutingOptions): AsyncIterable /** * Store the passed value under the passed key on the DHT */ - put(key: Uint8Array, value: Uint8Array, options?: QueryOptions): AsyncIterable + put(key: Uint8Array, value: Uint8Array, options?: RoutingOptions): AsyncIterable /** * Returns the mode this node is in */ - getMode(): Promise<'client' | 'server'> + getMode(): 'client' | 'server' /** - * If 'server' this node will respond to DHT queries, if 'client' this node will not + * If 'server' this node will respond to DHT queries, if 'client' this node + * will not. */ setMode(mode: 'client' | 'server'): Promise @@ -197,11 +280,6 @@ export interface SingleKadDHT extends KadDHT { routingTable: RoutingTable } -export interface DualKadDHT extends KadDHT { - wan: SingleKadDHT - lan: SingleKadDHT -} - /** * A selector function takes a DHT key and a list of records and returns the * index of the best record in the list @@ -226,12 +304,17 @@ export type Validators = Record export interface KadDHTInit { /** - * How many peers to store in each kBucket (default 20) + * How many peers to store in each kBucket + * + * @default 20 */ kBucketSize?: number /** - * Whether to start up as a DHT client or server + * If true, only ever be a DHT client. If false, be a DHT client until told + * to be a DHT server via `setMode`. + * + * @default false */ clientMode?: boolean @@ -254,7 +337,9 @@ export interface KadDHTInit { /** * During startup we run the self-query at a shorter interval to ensure * the containing node can respond to queries quickly. Set this interval - * here in ms (default: 1000) + * here in ms. + * + * @default 1000 */ initialQuerySelfInterval?: number @@ -262,34 +347,55 @@ export interface KadDHTInit { * After startup by default all queries will be paused until the initial * self-query has run and there are some peers in the routing table. * - * Pass true here to disable this behaviour. (default: false) + * Pass true here to disable this behaviour. + * + * @default false */ allowQueryWithZeroPeers?: boolean /** - * A custom protocol prefix to use (default: '/ipfs') + * The network protocol to use + * + * @default "/ipfs/kad/1.0.0" */ - protocolPrefix?: string + protocol?: string + + /** + * The logging prefix to use + * + * @default "libp2p:kad-dht" + */ + logPrefix?: string /** * How long to wait in ms when pinging DHT peers to decide if they - * should be evicted from the routing table or not (default 10000) + * should be evicted from the routing table or not. + * + * @default 10000 */ pingTimeout?: number /** * How many peers to ping in parallel when deciding if they should - * be evicted from the routing table or not (default 10) + * be evicted from the routing table or not + * + * @default 10 */ pingConcurrency?: number /** - * How many parallel incoming streams to allow on the DHT protocol per-connection + * How many parallel incoming streams to allow on the DHT protocol per + * connection + * + * @default 32 */ maxInboundStreams?: number /** - * How many parallel outgoing streams to allow on the DHT protocol per-connection + * How many parallel outgoing streams to allow on the DHT protocol per + * connection + * + * @default 64 */ maxOutboundStreams?: number @@ -297,6 +403,12 @@ export interface KadDHTInit { * Initialization options for the Providers component */ providers?: ProvidersInit + + /** + * For every incoming and outgoing PeerInfo, override address configuration + * with this filter. + */ + peerInfoMapper?(peer: PeerInfo): PeerInfo } export interface KadDHTComponents { @@ -311,6 +423,10 @@ export interface KadDHTComponents { logger: ComponentLogger } -export function kadDHT (init?: KadDHTInit): (components: KadDHTComponents) => DualKadDHT { - return (components: KadDHTComponents) => new DefaultDualKadDHT(components, init) +/** + * Creates a custom DHT implementation, please ensure you pass a `protocol` + * string as an option. + */ +export function kadDHT (init: KadDHTInit): (components: KadDHTComponents) => KadDHT { + return (components: KadDHTComponents) => new KadDHTClass(components, init) } diff --git a/packages/kad-dht/src/kad-dht.ts b/packages/kad-dht/src/kad-dht.ts index 8bf2dd759a..5f076d2c5c 100644 --- a/packages/kad-dht/src/kad-dht.ts +++ b/packages/kad-dht/src/kad-dht.ts @@ -1,10 +1,13 @@ -import { CustomEvent, TypedEventEmitter } from '@libp2p/interface' +import { CodeError, CustomEvent, TypedEventEmitter, contentRoutingSymbol, peerDiscoverySymbol, peerRoutingSymbol } from '@libp2p/interface' +import drain from 'it-drain' +import map from 'it-map' +import parallel from 'it-parallel' import pDefer from 'p-defer' -import { PROTOCOL_DHT, PROTOCOL_PREFIX, LAN_PREFIX } from './constants.js' +import { PROTOCOL } from './constants.js' import { ContentFetching } from './content-fetching/index.js' -import { ContentRouting } from './content-routing/index.js' +import { ContentRouting as KADDHTContentRouting } from './content-routing/index.js' import { Network } from './network.js' -import { PeerRouting } from './peer-routing/index.js' +import { PeerRouting as KADDHTPeerRouting } from './peer-routing/index.js' import { Providers } from './providers.js' import { QueryManager } from './query/manager.js' import { QuerySelf } from './query-self.js' @@ -15,45 +18,169 @@ import { RoutingTableRefresh } from './routing-table/refresh.js' import { RPC } from './rpc/index.js' import { TopologyListener } from './topology-listener.js' import { - removePrivateAddresses, - removePublicAddresses + multiaddrIsPublic, + removePrivateAddressesMapper } from './utils.js' -import type { KadDHTComponents, KadDHTInit, QueryOptions, Validators, Selectors, KadDHT, QueryEvent } from './index.js' -import type { Logger, PeerDiscoveryEvents, PeerId, PeerInfo } from '@libp2p/interface' +import type { KadDHTComponents, KadDHTInit, Validators, Selectors, KadDHT as KadDHTInterface, QueryEvent, PeerInfoMapper } from './index.js' +import type { AbortOptions, ContentRouting, Logger, PeerDiscovery, PeerDiscoveryEvents, PeerId, PeerInfo, PeerRouting, RoutingOptions, Startable } from '@libp2p/interface' import type { CID } from 'multiformats/cid' -export const DEFAULT_MAX_INBOUND_STREAMS = 32 -export const DEFAULT_MAX_OUTBOUND_STREAMS = 64 +async function * ensurePeerInfoHasMultiaddrs (source: AsyncGenerator, peerRouting: PeerRouting, log: Logger, options: AbortOptions = {}): AsyncGenerator<() => Promise, void, undefined> { + yield * map(source, prov => { + return async () => { + if (prov.multiaddrs.length > 0) { + return prov + } + + try { + return await peerRouting.findPeer(prov.id, { + ...options, + useCache: false + }) + } catch (err) { + log.error('could not find peer', err) + } + } + }) +} -export interface SingleKadDHTInit extends KadDHTInit { - /** - * Whether to start up in lan or wan mode - */ - lan?: boolean +/** + * Wrapper class to convert events into returned values + */ +class DHTContentRouting implements ContentRouting { + private readonly dht: KadDHTInterface + private readonly peerInfoMapper: PeerInfoMapper + private readonly peerRouting: PeerRouting + private readonly log: Logger + + constructor (dht: KadDHTInterface, peerInfoMapper: PeerInfoMapper, peerRouting: PeerRouting, log: Logger) { + this.dht = dht + this.peerInfoMapper = peerInfoMapper + this.peerRouting = peerRouting + this.log = log + } + + async provide (cid: CID, options: RoutingOptions = {}): Promise { + await drain(this.dht.provide(cid, options)) + } + + async * findProviders (cid: CID, options: RoutingOptions = {}): AsyncGenerator { + const self = this + const source = async function * (): AsyncGenerator { + for await (const event of self.dht.findProviders(cid, options)) { + if (event.name === 'PROVIDER') { + yield * event.providers + } + } + } + for await (let peerInfo of parallel(ensurePeerInfoHasMultiaddrs(source(), this.peerRouting, this.log, options))) { + if (peerInfo == null) { + continue + } + + peerInfo = this.peerInfoMapper(peerInfo) + + if (peerInfo.multiaddrs.length === 0) { + continue + } + + yield peerInfo + } + } + + async put (key: Uint8Array, value: Uint8Array, options?: RoutingOptions): Promise { + await drain(this.dht.put(key, value, options)) + } + + async get (key: Uint8Array, options?: RoutingOptions): Promise { + for await (const event of this.dht.get(key, options)) { + if (event.name === 'VALUE') { + return event.value + } + } + + throw new CodeError('Not found', 'ERR_NOT_FOUND') + } } +/** + * Wrapper class to convert events into returned values + */ +class DHTPeerRouting implements PeerRouting { + private readonly dht: KadDHTInterface + private readonly peerInfoMapper: PeerInfoMapper + private readonly log: Logger + + constructor (dht: KadDHTInterface, peerInfoMapper: PeerInfoMapper, log: Logger) { + this.dht = dht + this.peerInfoMapper = peerInfoMapper + this.log = log + } + + async findPeer (peerId: PeerId, options: RoutingOptions = {}): Promise { + for await (const event of this.dht.findPeer(peerId, options)) { + if (event.name === 'FINAL_PEER') { + const peer = this.peerInfoMapper(event.peer) + + if (peer.multiaddrs.length > 0) { + return event.peer + } + } + } + + throw new CodeError('Not found', 'ERR_NOT_FOUND') + } + + async * getClosestPeers (key: Uint8Array, options: RoutingOptions = {}): AsyncIterable { + const self = this + const source = async function * (): AsyncGenerator { + for await (const event of self.dht.getClosestPeers(key, options)) { + if (event.name === 'FINAL_PEER') { + yield event.peer + } + } + } + + for await (let peerInfo of parallel(ensurePeerInfoHasMultiaddrs(source(), this, this.log, options))) { + if (peerInfo == null) { + continue + } + + peerInfo = this.peerInfoMapper(peerInfo) + + if (peerInfo.multiaddrs.length === 0) { + continue + } + + yield peerInfo + } + } +} + +export const DEFAULT_MAX_INBOUND_STREAMS = 32 +export const DEFAULT_MAX_OUTBOUND_STREAMS = 64 + /** * A DHT implementation modelled after Kademlia with S/Kademlia modifications. * Original implementation in go: https://github.com/libp2p/go-libp2p-kad-dht. */ -export class DefaultKadDHT extends TypedEventEmitter implements KadDHT { +export class KadDHT extends TypedEventEmitter implements KadDHTInterface, Startable { public protocol: string public routingTable: RoutingTable public providers: Providers public network: Network - public peerRouting: PeerRouting + public peerRouting: KADDHTPeerRouting public readonly components: KadDHTComponents private readonly log: Logger private running: boolean private readonly kBucketSize: number private clientMode: boolean - private readonly lan: boolean private readonly validators: Validators private readonly selectors: Selectors private readonly queryManager: QueryManager private readonly contentFetching: ContentFetching - private readonly contentRouting: ContentRouting + private readonly contentRouting: KADDHTContentRouting private readonly routingTableRefresh: RoutingTableRefresh private readonly rpc: RPC private readonly topologyListener: TopologyListener @@ -61,10 +188,14 @@ export class DefaultKadDHT extends TypedEventEmitter implem private readonly maxInboundStreams: number private readonly maxOutboundStreams: number + private readonly dhtContentRouting: DHTContentRouting + private readonly dhtPeerRouting: DHTPeerRouting + private readonly peerInfoMapper: PeerInfoMapper + /** * Create a new KadDHT */ - constructor (components: KadDHTComponents, init: SingleKadDHTInit) { + constructor (components: KadDHTComponents, init: KadDHTInit) { super() const { @@ -73,8 +204,8 @@ export class DefaultKadDHT extends TypedEventEmitter implem validators, selectors, querySelfInterval, - lan, - protocolPrefix, + protocol, + logPrefix, pingTimeout, pingConcurrency, maxInboundStreams, @@ -82,21 +213,23 @@ export class DefaultKadDHT extends TypedEventEmitter implem providers: providersInit } = init + const loggingPrefix = logPrefix ?? 'libp2p:kad-dht' + this.running = false this.components = components - this.lan = Boolean(lan) - this.log = components.logger.forComponent(`libp2p:kad-dht:${lan === true ? 'lan' : 'wan'}`) - this.protocol = `${protocolPrefix ?? PROTOCOL_PREFIX}${lan === true ? LAN_PREFIX : ''}${PROTOCOL_DHT}` + this.log = components.logger.forComponent(loggingPrefix) + this.protocol = protocol ?? PROTOCOL this.kBucketSize = kBucketSize ?? 20 this.clientMode = clientMode ?? true this.maxInboundStreams = maxInboundStreams ?? DEFAULT_MAX_INBOUND_STREAMS this.maxOutboundStreams = maxOutboundStreams ?? DEFAULT_MAX_OUTBOUND_STREAMS + this.peerInfoMapper = init.peerInfoMapper ?? removePrivateAddressesMapper this.routingTable = new RoutingTable(components, { kBucketSize, - lan: this.lan, pingTimeout, pingConcurrency, - protocol: this.protocol + protocol: this.protocol, + logPrefix: loggingPrefix }) this.providers = new Providers(components, providersInit ?? {}) @@ -111,7 +244,7 @@ export class DefaultKadDHT extends TypedEventEmitter implem } this.network = new Network(components, { protocol: this.protocol, - lan: this.lan + logPrefix: loggingPrefix }) // all queries should wait for the initial query-self query to run so we have @@ -127,18 +260,18 @@ export class DefaultKadDHT extends TypedEventEmitter implem this.queryManager = new QueryManager(components, { // Number of disjoint query paths to use - This is set to `kBucketSize/2` per the S/Kademlia paper disjointPaths: Math.ceil(this.kBucketSize / 2), - lan, + logPrefix: loggingPrefix, initialQuerySelfHasRun, routingTable: this.routingTable }) // DHT components - this.peerRouting = new PeerRouting(components, { + this.peerRouting = new KADDHTPeerRouting(components, { routingTable: this.routingTable, network: this.network, validators: this.validators, queryManager: this.queryManager, - lan: this.lan + logPrefix: loggingPrefix }) this.contentFetching = new ContentFetching(components, { validators: this.validators, @@ -146,37 +279,38 @@ export class DefaultKadDHT extends TypedEventEmitter implem peerRouting: this.peerRouting, queryManager: this.queryManager, network: this.network, - lan: this.lan + logPrefix: loggingPrefix }) - this.contentRouting = new ContentRouting(components, { + this.contentRouting = new KADDHTContentRouting(components, { network: this.network, peerRouting: this.peerRouting, queryManager: this.queryManager, routingTable: this.routingTable, providers: this.providers, - lan: this.lan + logPrefix: loggingPrefix }) this.routingTableRefresh = new RoutingTableRefresh(components, { peerRouting: this.peerRouting, routingTable: this.routingTable, - lan: this.lan + logPrefix: loggingPrefix }) this.rpc = new RPC(components, { routingTable: this.routingTable, providers: this.providers, peerRouting: this.peerRouting, validators: this.validators, - lan: this.lan + logPrefix: loggingPrefix, + peerInfoMapper: this.peerInfoMapper }) this.topologyListener = new TopologyListener(components, { protocol: this.protocol, - lan: this.lan + logPrefix: loggingPrefix }) this.querySelf = new QuerySelf(components, { peerRouting: this.peerRouting, interval: querySelfInterval, initialInterval: init.initialQuerySelfInterval, - lan: this.lan, + logPrefix: loggingPrefix, initialQuerySelfHasRun, routingTable: this.routingTable }) @@ -212,19 +346,55 @@ export class DefaultKadDHT extends TypedEventEmitter implem this.log.error('could not add %p to routing table', peerId, err) }) }) + + this.dhtPeerRouting = new DHTPeerRouting(this, this.peerInfoMapper, this.log) + this.dhtContentRouting = new DHTContentRouting(this, this.peerInfoMapper, this.dhtPeerRouting, this.log) + + // if client mode has not been explicitly specified, auto-switch to server + // mode when the node's peer data is updated with publicly dialable + // addresses + if (init.clientMode == null) { + components.events.addEventListener('self:peer:update', (evt) => { + this.log('received update of self-peer info') + + void Promise.resolve().then(async () => { + const hasPublicAddress = evt.detail.peer.addresses + .some(({ multiaddr }) => multiaddrIsPublic(multiaddr)) + + const mode = this.getMode() + + if (hasPublicAddress && mode === 'client') { + await this.setMode('server') + } else if (mode === 'server' && !hasPublicAddress) { + await this.setMode('client') + } + }) + .catch(err => { + this.log.error('error setting dht server mode', err) + }) + }) + } + } + + get [contentRoutingSymbol] (): ContentRouting { + return this.dhtContentRouting + } + + get [peerRoutingSymbol] (): PeerRouting { + return this.dhtPeerRouting + } + + get [peerDiscoverySymbol] (): PeerDiscovery { + return this } async onPeerConnect (peerData: PeerInfo): Promise { this.log('peer %p connected', peerData.id) - if (this.lan) { - peerData = removePublicAddresses(peerData) - } else { - peerData = removePrivateAddresses(peerData) - } + peerData = this.peerInfoMapper(peerData) if (peerData.multiaddrs.length === 0) { - this.log('ignoring %p as they do not have any %s addresses in %s', peerData.id, this.lan ? 'private' : 'public', peerData.multiaddrs.map(addr => addr.toString())) + this.log('ignoring %p as there were no valid addresses in %s after filtering', peerData.id, peerData.multiaddrs.map(addr => addr.toString())) return } @@ -245,7 +415,7 @@ export class DefaultKadDHT extends TypedEventEmitter implem /** * If 'server' this node will respond to DHT queries, if 'client' this node will not */ - async getMode (): Promise<'client' | 'server'> { + getMode (): 'client' | 'server' { return this.clientMode ? 'client' : 'server' } @@ -277,17 +447,16 @@ export class DefaultKadDHT extends TypedEventEmitter implem // Only respond to queries when not in client mode await this.setMode(this.clientMode ? 'client' : 'server') + this.querySelf.start() + await Promise.all([ this.providers.start(), this.queryManager.start(), this.network.start(), this.routingTable.start(), - this.topologyListener.start() + this.topologyListener.start(), + this.routingTableRefresh.start() ]) - - this.querySelf.start() - - await this.routingTableRefresh.start() } /** @@ -312,14 +481,14 @@ export class DefaultKadDHT extends TypedEventEmitter implem /** * Store the given key/value pair in the DHT */ - async * put (key: Uint8Array, value: Uint8Array, options: QueryOptions = {}): AsyncGenerator { + async * put (key: Uint8Array, value: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { yield * this.contentFetching.put(key, value, options) } /** * Get the value that corresponds to the passed key */ - async * get (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator { + async * get (key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { yield * this.contentFetching.get(key, options) } @@ -328,14 +497,14 @@ export class DefaultKadDHT extends TypedEventEmitter implem /** * Announce to the network that we can provide given key's value */ - async * provide (key: CID, options: QueryOptions = {}): AsyncGenerator { + async * provide (key: CID, options: RoutingOptions = {}): AsyncGenerator { yield * this.contentRouting.provide(key, this.components.addressManager.getAddresses(), options) } /** * Search the dht for providers of the given CID */ - async * findProviders (key: CID, options: QueryOptions = {}): AsyncGenerator { + async * findProviders (key: CID, options: RoutingOptions = {}): AsyncGenerator { yield * this.contentRouting.findProviders(key, options) } @@ -344,14 +513,14 @@ export class DefaultKadDHT extends TypedEventEmitter implem /** * Search for a peer with the given ID */ - async * findPeer (id: PeerId, options: QueryOptions = {}): AsyncGenerator { + async * findPeer (id: PeerId, options: RoutingOptions = {}): AsyncGenerator { yield * this.peerRouting.findPeer(id, options) } /** * Kademlia 'node lookup' operation */ - async * getClosestPeers (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator { + async * getClosestPeers (key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { yield * this.peerRouting.getClosestPeers(key, options) } diff --git a/packages/kad-dht/src/message/dht.proto b/packages/kad-dht/src/message/dht.proto index 160735b952..40657b9109 100644 --- a/packages/kad-dht/src/message/dht.proto +++ b/packages/kad-dht/src/message/dht.proto @@ -12,48 +12,48 @@ message Record { optional string timeReceived = 5; } -message Message { - enum MessageType { - PUT_VALUE = 0; - GET_VALUE = 1; - ADD_PROVIDER = 2; - GET_PROVIDERS = 3; - FIND_NODE = 4; - PING = 5; - } +enum MessageType { + PUT_VALUE = 0; + GET_VALUE = 1; + ADD_PROVIDER = 2; + GET_PROVIDERS = 3; + FIND_NODE = 4; + PING = 5; +} - enum ConnectionType { - // sender does not have a connection to peer, and no extra information (default) - NOT_CONNECTED = 0; +enum ConnectionType { + // sender does not have a connection to peer, and no extra information (default) + NOT_CONNECTED = 0; - // sender has a live connection to peer - CONNECTED = 1; + // sender has a live connection to peer + CONNECTED = 1; - // sender recently connected to peer - CAN_CONNECT = 2; + // sender recently connected to peer + CAN_CONNECT = 2; - // sender recently tried to connect to peer repeatedly but failed to connect - // ("try" here is loose, but this should signal "made strong effort, failed") - CANNOT_CONNECT = 3; - } + // sender recently tried to connect to peer repeatedly but failed to connect + // ("try" here is loose, but this should signal "made strong effort, failed") + CANNOT_CONNECT = 3; +} - message Peer { - // ID of a given peer. - optional bytes id = 1; +message PeerInfo { + // ID of a given peer. + bytes id = 1; - // multiaddrs for a given peer - repeated bytes addrs = 2; + // multiaddrs for a given peer + repeated bytes multiaddrs = 2; - // used to signal the sender's connection capabilities to the peer - optional ConnectionType connection = 3; - } + // used to signal the sender's connection capabilities to the peer + optional ConnectionType connection = 3; +} +message Message { // defines what type of message it is. - optional MessageType type = 1; + MessageType type = 1; // defines what coral cluster level this query/response belongs to. // in case we want to implement coral's cluster rings in the future. - optional int32 clusterLevelRaw = 10; + optional int32 clusterLevel = 10; // Used to specify the key associated with this message. // PUT_VALUE, GET_VALUE, ADD_PROVIDER, GET_PROVIDERS @@ -67,9 +67,9 @@ message Message { // Used to return peers closer to a key in a query // GET_VALUE, GET_PROVIDERS, FIND_NODE - repeated Peer closerPeers = 8; + repeated PeerInfo closer = 8; // Used to return Providers // GET_VALUE, ADD_PROVIDER, GET_PROVIDERS - repeated Peer providerPeers = 9; + repeated PeerInfo providers = 9; } diff --git a/packages/kad-dht/src/message/dht.ts b/packages/kad-dht/src/message/dht.ts index af919307e3..3dfcefbf0f 100644 --- a/packages/kad-dht/src/message/dht.ts +++ b/packages/kad-dht/src/message/dht.ts @@ -4,8 +4,8 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message, enumeration } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' import type { Uint8ArrayList } from 'uint8arraylist' export interface Record { @@ -63,24 +63,30 @@ export namespace Record { const tag = reader.uint32() switch (tag >>> 3) { - case 1: + case 1: { obj.key = reader.bytes() break - case 2: + } + case 2: { obj.value = reader.bytes() break - case 3: + } + case 3: { obj.author = reader.bytes() break - case 4: + } + case 4: { obj.signature = reader.bytes() break - case 5: + } + case 5: { obj.timeReceived = reader.string() break - default: + } + default: { reader.skipType(tag & 7) break + } } } @@ -100,138 +106,141 @@ export namespace Record { } } -export interface Message { - type?: Message.MessageType - clusterLevelRaw?: number - key?: Uint8Array - record?: Uint8Array - closerPeers: Message.Peer[] - providerPeers: Message.Peer[] +export enum MessageType { + PUT_VALUE = 'PUT_VALUE', + GET_VALUE = 'GET_VALUE', + ADD_PROVIDER = 'ADD_PROVIDER', + GET_PROVIDERS = 'GET_PROVIDERS', + FIND_NODE = 'FIND_NODE', + PING = 'PING' } -export namespace Message { - export enum MessageType { - PUT_VALUE = 'PUT_VALUE', - GET_VALUE = 'GET_VALUE', - ADD_PROVIDER = 'ADD_PROVIDER', - GET_PROVIDERS = 'GET_PROVIDERS', - FIND_NODE = 'FIND_NODE', - PING = 'PING' - } - - enum __MessageTypeValues { - PUT_VALUE = 0, - GET_VALUE = 1, - ADD_PROVIDER = 2, - GET_PROVIDERS = 3, - FIND_NODE = 4, - PING = 5 - } +enum __MessageTypeValues { + PUT_VALUE = 0, + GET_VALUE = 1, + ADD_PROVIDER = 2, + GET_PROVIDERS = 3, + FIND_NODE = 4, + PING = 5 +} - export namespace MessageType { - export const codec = (): Codec => { - return enumeration(__MessageTypeValues) - } +export namespace MessageType { + export const codec = (): Codec => { + return enumeration(__MessageTypeValues) } +} +export enum ConnectionType { + NOT_CONNECTED = 'NOT_CONNECTED', + CONNECTED = 'CONNECTED', + CAN_CONNECT = 'CAN_CONNECT', + CANNOT_CONNECT = 'CANNOT_CONNECT' +} - export enum ConnectionType { - NOT_CONNECTED = 'NOT_CONNECTED', - CONNECTED = 'CONNECTED', - CAN_CONNECT = 'CAN_CONNECT', - CANNOT_CONNECT = 'CANNOT_CONNECT' - } +enum __ConnectionTypeValues { + NOT_CONNECTED = 0, + CONNECTED = 1, + CAN_CONNECT = 2, + CANNOT_CONNECT = 3 +} - enum __ConnectionTypeValues { - NOT_CONNECTED = 0, - CONNECTED = 1, - CAN_CONNECT = 2, - CANNOT_CONNECT = 3 +export namespace ConnectionType { + export const codec = (): Codec => { + return enumeration(__ConnectionTypeValues) } +} +export interface PeerInfo { + id: Uint8Array + multiaddrs: Uint8Array[] + connection?: ConnectionType +} - export namespace ConnectionType { - export const codec = (): Codec => { - return enumeration(__ConnectionTypeValues) - } - } +export namespace PeerInfo { + let _codec: Codec - export interface Peer { - id?: Uint8Array - addrs: Uint8Array[] - connection?: Message.ConnectionType - } + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } - export namespace Peer { - let _codec: Codec + if ((obj.id != null && obj.id.byteLength > 0)) { + w.uint32(10) + w.bytes(obj.id) + } - export const codec = (): Codec => { - if (_codec == null) { - _codec = message((obj, w, opts = {}) => { - if (opts.lengthDelimited !== false) { - w.fork() + if (obj.multiaddrs != null) { + for (const value of obj.multiaddrs) { + w.uint32(18) + w.bytes(value) } + } - if (obj.id != null) { - w.uint32(10) - w.bytes(obj.id) - } + if (obj.connection != null) { + w.uint32(24) + ConnectionType.codec().encode(obj.connection, w) + } - if (obj.addrs != null) { - for (const value of obj.addrs) { - w.uint32(18) - w.bytes(value) - } - } + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + id: uint8ArrayAlloc(0), + multiaddrs: [] + } - if (obj.connection != null) { - w.uint32(24) - Message.ConnectionType.codec().encode(obj.connection, w) - } + const end = length == null ? reader.len : reader.pos + length - if (opts.lengthDelimited !== false) { - w.ldelim() - } - }, (reader, length) => { - const obj: any = { - addrs: [] - } + while (reader.pos < end) { + const tag = reader.uint32() - const end = length == null ? reader.len : reader.pos + length - - while (reader.pos < end) { - const tag = reader.uint32() - - switch (tag >>> 3) { - case 1: - obj.id = reader.bytes() - break - case 2: - obj.addrs.push(reader.bytes()) - break - case 3: - obj.connection = Message.ConnectionType.codec().decode(reader) - break - default: - reader.skipType(tag & 7) - break + switch (tag >>> 3) { + case 1: { + obj.id = reader.bytes() + break + } + case 2: { + obj.multiaddrs.push(reader.bytes()) + break + } + case 3: { + obj.connection = ConnectionType.codec().decode(reader) + break + } + default: { + reader.skipType(tag & 7) + break } } + } - return obj - }) - } - - return _codec + return obj + }) } - export const encode = (obj: Partial): Uint8Array => { - return encodeMessage(obj, Peer.codec()) - } + return _codec + } - export const decode = (buf: Uint8Array | Uint8ArrayList): Peer => { - return decodeMessage(buf, Peer.codec()) - } + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, PeerInfo.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): PeerInfo => { + return decodeMessage(buf, PeerInfo.codec()) } +} + +export interface Message { + type: MessageType + clusterLevel?: number + key?: Uint8Array + record?: Uint8Array + closer: PeerInfo[] + providers: PeerInfo[] +} +export namespace Message { let _codec: Codec export const codec = (): Codec => { @@ -241,14 +250,14 @@ export namespace Message { w.fork() } - if (obj.type != null) { + if (obj.type != null && __MessageTypeValues[obj.type] !== 0) { w.uint32(8) - Message.MessageType.codec().encode(obj.type, w) + MessageType.codec().encode(obj.type, w) } - if (obj.clusterLevelRaw != null) { + if (obj.clusterLevel != null) { w.uint32(80) - w.int32(obj.clusterLevelRaw) + w.int32(obj.clusterLevel) } if (obj.key != null) { @@ -261,17 +270,17 @@ export namespace Message { w.bytes(obj.record) } - if (obj.closerPeers != null) { - for (const value of obj.closerPeers) { + if (obj.closer != null) { + for (const value of obj.closer) { w.uint32(66) - Message.Peer.codec().encode(value, w) + PeerInfo.codec().encode(value, w) } } - if (obj.providerPeers != null) { - for (const value of obj.providerPeers) { + if (obj.providers != null) { + for (const value of obj.providers) { w.uint32(74) - Message.Peer.codec().encode(value, w) + PeerInfo.codec().encode(value, w) } } @@ -280,8 +289,9 @@ export namespace Message { } }, (reader, length) => { const obj: any = { - closerPeers: [], - providerPeers: [] + type: MessageType.PUT_VALUE, + closer: [], + providers: [] } const end = length == null ? reader.len : reader.pos + length @@ -290,27 +300,34 @@ export namespace Message { const tag = reader.uint32() switch (tag >>> 3) { - case 1: - obj.type = Message.MessageType.codec().decode(reader) + case 1: { + obj.type = MessageType.codec().decode(reader) break - case 10: - obj.clusterLevelRaw = reader.int32() + } + case 10: { + obj.clusterLevel = reader.int32() break - case 2: + } + case 2: { obj.key = reader.bytes() break - case 3: + } + case 3: { obj.record = reader.bytes() break - case 8: - obj.closerPeers.push(Message.Peer.codec().decode(reader, reader.uint32())) + } + case 8: { + obj.closer.push(PeerInfo.codec().decode(reader, reader.uint32())) break - case 9: - obj.providerPeers.push(Message.Peer.codec().decode(reader, reader.uint32())) + } + case 9: { + obj.providers.push(PeerInfo.codec().decode(reader, reader.uint32())) break - default: + } + default: { reader.skipType(tag & 7) break + } } } diff --git a/packages/kad-dht/src/message/index.ts b/packages/kad-dht/src/message/index.ts deleted file mode 100644 index 2f3ca3b7c6..0000000000 --- a/packages/kad-dht/src/message/index.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { peerIdFromBytes } from '@libp2p/peer-id' -import { multiaddr } from '@multiformats/multiaddr' -import { Libp2pRecord } from '../record/index.js' -import { Message as PBMessage } from './dht.js' -import type { PeerInfo } from '@libp2p/interface' -import type { Uint8ArrayList } from 'uint8arraylist' - -export const MESSAGE_TYPE = PBMessage.MessageType -export const CONNECTION_TYPE = PBMessage.ConnectionType -export const MESSAGE_TYPE_LOOKUP = Object.keys(MESSAGE_TYPE) - -interface PBPeer { - id: Uint8Array - addrs: Uint8Array[] - connection: PBMessage.ConnectionType -} - -/** - * Represents a single DHT control message. - */ -export class Message { - public type: PBMessage.MessageType - public key: Uint8Array - private clusterLevelRaw: number - public closerPeers: PeerInfo[] - public providerPeers: PeerInfo[] - public record?: Libp2pRecord - - constructor (type: PBMessage.MessageType, key: Uint8Array, level: number) { - if (!(key instanceof Uint8Array)) { - throw new Error('Key must be a Uint8Array') - } - - this.type = type - this.key = key - this.clusterLevelRaw = level - this.closerPeers = [] - this.providerPeers = [] - this.record = undefined - } - - /** - * @type {number} - */ - get clusterLevel (): number { - const level = this.clusterLevelRaw - 1 - if (level < 0) { - return 0 - } - - return level - } - - set clusterLevel (level) { - this.clusterLevelRaw = level - } - - /** - * Encode into protobuf - */ - serialize (): Uint8Array { - return PBMessage.encode({ - key: this.key, - type: this.type, - clusterLevelRaw: this.clusterLevelRaw, - closerPeers: this.closerPeers.map(toPbPeer), - providerPeers: this.providerPeers.map(toPbPeer), - record: this.record == null ? undefined : this.record.serialize().subarray() - }) - } - - /** - * Decode from protobuf - */ - static deserialize (raw: Uint8ArrayList | Uint8Array): Message { - const dec = PBMessage.decode(raw) - - const msg = new Message(dec.type ?? PBMessage.MessageType.PUT_VALUE, dec.key ?? Uint8Array.from([]), dec.clusterLevelRaw ?? 0) - msg.closerPeers = dec.closerPeers.map(fromPbPeer) - msg.providerPeers = dec.providerPeers.map(fromPbPeer) - - if (dec.record?.length != null) { - msg.record = Libp2pRecord.deserialize(dec.record) - } - - return msg - } - - static encode (message: Message): Uint8Array { - return message.serialize() - } - - static decode (buf: Uint8Array | Uint8ArrayList): Message { - return Message.deserialize(buf) - } -} - -function toPbPeer (peer: PeerInfo): PBPeer { - const output: PBPeer = { - id: peer.id.toBytes(), - addrs: (peer.multiaddrs ?? []).map((m) => m.bytes), - connection: CONNECTION_TYPE.CONNECTED - } - - return output -} - -function fromPbPeer (peer: PBMessage.Peer): PeerInfo { - if (peer.id == null) { - throw new Error('Invalid peer in message') - } - - return { - id: peerIdFromBytes(peer.id), - multiaddrs: (peer.addrs ?? []).map((a) => multiaddr(a)) - } -} diff --git a/packages/kad-dht/src/message/utils.ts b/packages/kad-dht/src/message/utils.ts new file mode 100644 index 0000000000..51800a28c8 --- /dev/null +++ b/packages/kad-dht/src/message/utils.ts @@ -0,0 +1,25 @@ +import { peerIdFromBytes } from '@libp2p/peer-id' +import { multiaddr } from '@multiformats/multiaddr' +import type { PeerInfo as PBPeerInfo, ConnectionType } from './dht.js' +import type { PeerInfo } from '@libp2p/interface' + +export function toPbPeerInfo (peer: PeerInfo, connection?: ConnectionType): PBPeerInfo { + const output: PBPeerInfo = { + id: peer.id.toBytes(), + multiaddrs: (peer.multiaddrs ?? []).map((m) => m.bytes), + connection + } + + return output +} + +export function fromPbPeerInfo (peer: PBPeerInfo): PeerInfo { + if (peer.id == null) { + throw new Error('Invalid peer in message') + } + + return { + id: peerIdFromBytes(peer.id), + multiaddrs: (peer.multiaddrs ?? []).map((a) => multiaddr(a)) + } +} diff --git a/packages/kad-dht/src/network.ts b/packages/kad-dht/src/network.ts index 46dfa71251..7fd50069b5 100644 --- a/packages/kad-dht/src/network.ts +++ b/packages/kad-dht/src/network.ts @@ -1,18 +1,21 @@ -import { TypedEventEmitter, CustomEvent } from '@libp2p/interface' +import { TypedEventEmitter } from '@libp2p/interface' import { pbStream } from 'it-protobuf-stream' -import { Message } from './message/index.js' +import { CodeError } from 'protons-runtime' +import { Message } from './message/dht.js' +import { fromPbPeerInfo } from './message/utils.js' import { dialPeerEvent, sendQueryEvent, peerResponseEvent, queryErrorEvent } from './query/events.js' -import type { KadDHTComponents, QueryEvent, QueryOptions } from './index.js' -import type { AbortOptions, Logger, Stream, PeerId, PeerInfo, Startable } from '@libp2p/interface' +import { Libp2pRecord } from './record/index.js' +import type { KadDHTComponents, QueryEvent } from './index.js' +import type { AbortOptions, Logger, Stream, PeerId, PeerInfo, Startable, RoutingOptions } from '@libp2p/interface' export interface NetworkInit { protocol: string - lan: boolean + logPrefix: string } interface NetworkEvents { @@ -34,9 +37,9 @@ export class Network extends TypedEventEmitter implements Startab constructor (components: KadDHTComponents, init: NetworkInit) { super() - const { protocol, lan } = init + const { protocol } = init this.components = components - this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:network`) + this.log = components.logger.forComponent(`${init.logPrefix}:network`) this.running = false this.protocol = protocol } @@ -69,14 +72,20 @@ export class Network extends TypedEventEmitter implements Startab /** * Send a request and record RTT for latency measurements */ - async * sendRequest (to: PeerId, msg: Message, options: QueryOptions = {}): AsyncGenerator { + async * sendRequest (to: PeerId, msg: Partial, options: RoutingOptions = {}): AsyncGenerator { if (!this.running) { return } + const type = msg.type + + if (type == null) { + throw new CodeError('Message type was missing', 'ERR_INVALID_PARAMETERS') + } + this.log('sending %s to %p', msg.type, to) yield dialPeerEvent({ peer: to }, options) - yield sendQueryEvent({ to, type: msg.type }, options) + yield sendQueryEvent({ to, type }, options) let stream: Stream | undefined @@ -89,11 +98,12 @@ export class Network extends TypedEventEmitter implements Startab yield peerResponseEvent({ from: to, messageType: response.type, - closer: response.closerPeers, - providers: response.providerPeers, - record: response.record + closer: response.closer.map(fromPbPeerInfo), + providers: response.providers.map(fromPbPeerInfo), + record: response.record == null ? undefined : Libp2pRecord.deserialize(response.record) }, options) } catch (err: any) { + this.log.error('could not send %s to %p', msg.type, to, err) yield queryErrorEvent({ from: to, error: err }, options) } finally { if (stream != null) { @@ -105,14 +115,20 @@ export class Network extends TypedEventEmitter implements Startab /** * Sends a message without expecting an answer */ - async * sendMessage (to: PeerId, msg: Message, options: QueryOptions = {}): AsyncGenerator { + async * sendMessage (to: PeerId, msg: Partial, options: RoutingOptions = {}): AsyncGenerator { if (!this.running) { return } + const type = msg.type + + if (type == null) { + throw new CodeError('Message type was missing', 'ERR_INVALID_PARAMETERS') + } + this.log('sending %s to %p', msg.type, to) yield dialPeerEvent({ peer: to }, options) - yield sendQueryEvent({ to, type: msg.type }, options) + yield sendQueryEvent({ to, type }, options) let stream: Stream | undefined @@ -122,7 +138,7 @@ export class Network extends TypedEventEmitter implements Startab await this._writeMessage(stream, msg, options) - yield peerResponseEvent({ from: to, messageType: msg.type }, options) + yield peerResponseEvent({ from: to, messageType: type }, options) } catch (err: any) { yield queryErrorEvent({ from: to, error: err }, options) } finally { @@ -135,7 +151,7 @@ export class Network extends TypedEventEmitter implements Startab /** * Write a message to the given stream */ - async _writeMessage (stream: Stream, msg: Message, options: AbortOptions): Promise { + async _writeMessage (stream: Stream, msg: Partial, options: AbortOptions): Promise { const pb = pbStream(stream) await pb.write(msg, Message, options) await pb.unwrap().close(options) @@ -146,7 +162,7 @@ export class Network extends TypedEventEmitter implements Startab * If no response is received after the specified timeout * this will error out. */ - async _writeReadMessage (stream: Stream, msg: Message, options: AbortOptions): Promise { + async _writeReadMessage (stream: Stream, msg: Partial, options: AbortOptions): Promise { const pb = pbStream(stream) await pb.write(msg, Message, options) @@ -156,15 +172,15 @@ export class Network extends TypedEventEmitter implements Startab await pb.unwrap().close(options) // tell any listeners about new peers we've seen - message.closerPeers.forEach(peerData => { - this.dispatchEvent(new CustomEvent('peer', { - detail: peerData - })) + message.closer.forEach(peerData => { + this.safeDispatchEvent('peer', { + detail: fromPbPeerInfo(peerData) + }) }) - message.providerPeers.forEach(peerData => { - this.dispatchEvent(new CustomEvent('peer', { - detail: peerData - })) + message.providers.forEach(peerData => { + this.safeDispatchEvent('peer', { + detail: fromPbPeerInfo(peerData) + }) }) return message diff --git a/packages/kad-dht/src/peer-routing/index.ts b/packages/kad-dht/src/peer-routing/index.ts index df0f25d9db..d5d63fda38 100644 --- a/packages/kad-dht/src/peer-routing/index.ts +++ b/packages/kad-dht/src/peer-routing/index.ts @@ -2,7 +2,7 @@ import { keys } from '@libp2p/crypto' import { CodeError } from '@libp2p/interface' import { peerIdFromKeys } from '@libp2p/peer-id' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { Message, MESSAGE_TYPE } from '../message/index.js' +import { MessageType } from '../message/dht.js' import { PeerDistanceList } from '../peer-list/peer-distance-list.js' import { queryErrorEvent, @@ -13,18 +13,19 @@ import { Libp2pRecord } from '../record/index.js' import { verifyRecord } from '../record/validators.js' import * as utils from '../utils.js' import type { KadDHTComponents, DHTRecord, DialPeerEvent, FinalPeerEvent, QueryEvent, Validators } from '../index.js' +import type { Message } from '../message/dht.js' import type { Network } from '../network.js' import type { QueryManager, QueryOptions } from '../query/manager.js' import type { QueryFunc } from '../query/types.js' import type { RoutingTable } from '../routing-table/index.js' -import type { AbortOptions, Logger, PeerId, PeerInfo, PeerStore } from '@libp2p/interface' +import type { Logger, PeerId, PeerInfo, PeerStore, RoutingOptions } from '@libp2p/interface' export interface PeerRoutingInit { routingTable: RoutingTable network: Network validators: Validators queryManager: QueryManager - lan: boolean + logPrefix: string } export class PeerRouting { @@ -37,7 +38,7 @@ export class PeerRouting { private readonly peerId: PeerId constructor (components: KadDHTComponents, init: PeerRoutingInit) { - const { routingTable, network, validators, queryManager, lan } = init + const { routingTable, network, validators, queryManager, logPrefix } = init this.routingTable = routingTable this.network = network @@ -45,7 +46,7 @@ export class PeerRouting { this.queryManager = queryManager this.peerStore = components.peerStore this.peerId = components.peerId - this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:peer-routing`) + this.log = components.logger.forComponent(`${logPrefix}:peer-routing`) } /** @@ -93,15 +94,19 @@ export class PeerRouting { /** * Get a value via rpc call for the given parameters */ - async * _getValueSingle (peer: PeerId, key: Uint8Array, options: AbortOptions = {}): AsyncGenerator { - const msg = new Message(MESSAGE_TYPE.GET_VALUE, key, 0) + async * _getValueSingle (peer: PeerId, key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { + const msg: Partial = { + type: MessageType.GET_VALUE, + key + } + yield * this.network.sendRequest(peer, msg, options) } /** * Get the public key directly from a node */ - async * getPublicKeyFromNode (peer: PeerId, options: AbortOptions = {}): AsyncGenerator { + async * getPublicKeyFromNode (peer: PeerId, options: RoutingOptions = {}): AsyncGenerator { const pkKey = utils.keyForPublicKey(peer) for await (const event of this._getValueSingle(peer, pkKey, options)) { @@ -129,52 +134,59 @@ export class PeerRouting { /** * Search for a peer with the given ID */ - async * findPeer (id: PeerId, options: QueryOptions = {}): AsyncGenerator { + async * findPeer (id: PeerId, options: RoutingOptions = {}): AsyncGenerator { this.log('findPeer %p', id) - // Try to find locally - const pi = await this.findPeerLocal(id) - - // already got it - if (pi != null) { - this.log('found local') - yield finalPeerEvent({ - from: this.peerId, - peer: pi - }, options) - return + if (options.useCache !== false) { + // Try to find locally + const pi = await this.findPeerLocal(id) + + // already got it + if (pi != null) { + this.log('found local') + yield finalPeerEvent({ + from: this.peerId, + peer: pi + }, options) + return + } } - const self = this // eslint-disable-line @typescript-eslint/no-this-alias + let foundPeer = false - const findPeerQuery: QueryFunc = async function * ({ peer, signal }) { - const request = new Message(MESSAGE_TYPE.FIND_NODE, id.toBytes(), 0) + if (options.useNetwork !== false) { + const self = this // eslint-disable-line @typescript-eslint/no-this-alias - for await (const event of self.network.sendRequest(peer, request, { - ...options, - signal - })) { - yield event + const findPeerQuery: QueryFunc = async function * ({ peer, signal }) { + const request: Partial = { + type: MessageType.FIND_NODE, + key: id.toBytes() + } + + for await (const event of self.network.sendRequest(peer, request, { + ...options, + signal + })) { + yield event - if (event.name === 'PEER_RESPONSE') { - const match = event.closer.find((p) => p.id.equals(id)) + if (event.name === 'PEER_RESPONSE') { + const match = event.closer.find((p) => p.id.equals(id)) - // found the peer - if (match != null) { - yield finalPeerEvent({ from: event.from, peer: match }, options) + // found the peer + if (match != null) { + yield finalPeerEvent({ from: event.from, peer: match }, options) + } } } } - } - let foundPeer = false + for await (const event of this.queryManager.run(id.toBytes(), findPeerQuery, options)) { + if (event.name === 'FINAL_PEER') { + foundPeer = true + } - for await (const event of this.queryManager.run(id.toBytes(), findPeerQuery, options)) { - if (event.name === 'FINAL_PEER') { - foundPeer = true + yield event } - - yield event } if (!foundPeer) { @@ -197,7 +209,10 @@ export class PeerRouting { const getCloserPeersQuery: QueryFunc = async function * ({ peer, signal }) { self.log('closerPeersSingle %s from %p', uint8ArrayToString(key, 'base32'), peer) - const request = new Message(MESSAGE_TYPE.FIND_NODE, key, 0) + const request: Partial = { + type: MessageType.FIND_NODE, + key + } yield * self.network.sendRequest(peer, request, { ...options, @@ -240,7 +255,7 @@ export class PeerRouting { * * Note: The peerStore is updated with new addresses found for the given peer. */ - async * getValueOrPeers (peer: PeerId, key: Uint8Array, options: AbortOptions = {}): AsyncGenerator { + async * getValueOrPeers (peer: PeerId, key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator { for await (const event of this._getValueSingle(peer, key, options)) { if (event.name === 'PEER_RESPONSE') { if (event.record != null) { diff --git a/packages/kad-dht/src/providers.ts b/packages/kad-dht/src/providers.ts index e7c6a9439b..e068e54b59 100644 --- a/packages/kad-dht/src/providers.ts +++ b/packages/kad-dht/src/providers.ts @@ -15,13 +15,20 @@ import type { Datastore } from 'interface-datastore' import type { CID } from 'multiformats' export interface ProvidersInit { + /** + * @default 256 + */ cacheSize?: number /** * How often invalid records are cleaned. (in seconds) + * + * @default 5400 */ cleanupInterval?: number /** * How long is a provider valid for. (in seconds) + * + * @default 86400 */ provideValidity?: number } diff --git a/packages/kad-dht/src/query-self.ts b/packages/kad-dht/src/query-self.ts index d0d5ae17f8..7b3bee924d 100644 --- a/packages/kad-dht/src/query-self.ts +++ b/packages/kad-dht/src/query-self.ts @@ -12,7 +12,7 @@ import type { ComponentLogger, Logger, PeerId, Startable } from '@libp2p/interfa import type { DeferredPromise } from 'p-defer' export interface QuerySelfInit { - lan: boolean + logPrefix: string peerRouting: PeerRouting routingTable: RoutingTable count?: number @@ -46,10 +46,10 @@ export class QuerySelf implements Startable { private querySelfPromise?: DeferredPromise constructor (components: QuerySelfComponents, init: QuerySelfInit) { - const { peerRouting, lan, count, interval, queryTimeout, routingTable } = init + const { peerRouting, logPrefix, count, interval, queryTimeout, routingTable } = init this.peerId = components.peerId - this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:query-self`) + this.log = components.logger.forComponent(`${logPrefix}:query-self`) this.started = false this.peerRouting = peerRouting this.routingTable = routingTable diff --git a/packages/kad-dht/src/query/events.ts b/packages/kad-dht/src/query/events.ts index 40c28bd397..da7cbb7f63 100644 --- a/packages/kad-dht/src/query/events.ts +++ b/packages/kad-dht/src/query/events.ts @@ -1,22 +1,22 @@ import { CustomEvent } from '@libp2p/interface' -import { MESSAGE_TYPE_LOOKUP } from '../message/index.js' -import type { SendQueryEvent, PeerResponseEvent, DialPeerEvent, AddPeerEvent, ValueEvent, ProviderEvent, QueryErrorEvent, FinalPeerEvent, QueryOptions } from '../index.js' -import type { Message } from '../message/dht.js' +import { MessageType } from '../message/dht.js' +import type { SendQueryEvent, PeerResponseEvent, DialPeerEvent, AddPeerEvent, ValueEvent, ProviderEvent, QueryErrorEvent, FinalPeerEvent } from '../index.js' import type { Libp2pRecord } from '../record/index.js' import type { PeerId, PeerInfo } from '@libp2p/interface' +import type { ProgressOptions } from 'progress-events' export interface QueryEventFields { to: PeerId - type: Message.MessageType + type: MessageType } -export function sendQueryEvent (fields: QueryEventFields, options: QueryOptions = {}): SendQueryEvent { +export function sendQueryEvent (fields: QueryEventFields, options: ProgressOptions = {}): SendQueryEvent { const event: SendQueryEvent = { ...fields, name: 'SEND_QUERY', type: 0, messageName: fields.type, - messageType: MESSAGE_TYPE_LOOKUP.indexOf(fields.type.toString()) + messageType: MessageType[fields.type] } options.onProgress?.(new CustomEvent('kad-dht:query:send-query', { detail: event })) @@ -26,13 +26,13 @@ export function sendQueryEvent (fields: QueryEventFields, options: QueryOptions export interface PeerResponseEventField { from: PeerId - messageType: Message.MessageType + messageType: MessageType closer?: PeerInfo[] providers?: PeerInfo[] record?: Libp2pRecord } -export function peerResponseEvent (fields: PeerResponseEventField, options: QueryOptions = {}): PeerResponseEvent { +export function peerResponseEvent (fields: PeerResponseEventField, options: ProgressOptions = {}): PeerResponseEvent { const event: PeerResponseEvent = { ...fields, name: 'PEER_RESPONSE', @@ -52,7 +52,7 @@ export interface FinalPeerEventFields { peer: PeerInfo } -export function finalPeerEvent (fields: FinalPeerEventFields, options: QueryOptions = {}): FinalPeerEvent { +export function finalPeerEvent (fields: FinalPeerEventFields, options: ProgressOptions = {}): FinalPeerEvent { const event: FinalPeerEvent = { ...fields, name: 'FINAL_PEER', @@ -69,7 +69,7 @@ export interface ErrorEventFields { error: Error } -export function queryErrorEvent (fields: ErrorEventFields, options: QueryOptions = {}): QueryErrorEvent { +export function queryErrorEvent (fields: ErrorEventFields, options: ProgressOptions = {}): QueryErrorEvent { const event: QueryErrorEvent = { ...fields, name: 'QUERY_ERROR', @@ -86,7 +86,7 @@ export interface ProviderEventFields { providers: PeerInfo[] } -export function providerEvent (fields: ProviderEventFields, options: QueryOptions = {}): ProviderEvent { +export function providerEvent (fields: ProviderEventFields, options: ProgressOptions = {}): ProviderEvent { const event: ProviderEvent = { ...fields, name: 'PROVIDER', @@ -103,7 +103,7 @@ export interface ValueEventFields { value: Uint8Array } -export function valueEvent (fields: ValueEventFields, options: QueryOptions = {}): ValueEvent { +export function valueEvent (fields: ValueEventFields, options: ProgressOptions = {}): ValueEvent { const event: ValueEvent = { ...fields, name: 'VALUE', @@ -119,7 +119,7 @@ export interface PeerEventFields { peer: PeerId } -export function addPeerEvent (fields: PeerEventFields, options: QueryOptions = {}): AddPeerEvent { +export function addPeerEvent (fields: PeerEventFields, options: ProgressOptions = {}): AddPeerEvent { const event: AddPeerEvent = { ...fields, name: 'ADD_PEER', @@ -135,7 +135,7 @@ export interface DialPeerEventFields { peer: PeerId } -export function dialPeerEvent (fields: DialPeerEventFields, options: QueryOptions = {}): DialPeerEvent { +export function dialPeerEvent (fields: DialPeerEventFields, options: ProgressOptions = {}): DialPeerEvent { const event: DialPeerEvent = { ...fields, name: 'DIAL_PEER', diff --git a/packages/kad-dht/src/query/manager.ts b/packages/kad-dht/src/query/manager.ts index 9e2ab435ff..3795b54843 100644 --- a/packages/kad-dht/src/query/manager.ts +++ b/packages/kad-dht/src/query/manager.ts @@ -9,9 +9,9 @@ import { import { convertBuffer } from '../utils.js' import { queryPath } from './query-path.js' import type { QueryFunc } from './types.js' -import type { QueryEvent, QueryOptions as RootQueryOptions } from '../index.js' +import type { QueryEvent } from '../index.js' import type { RoutingTable } from '../routing-table/index.js' -import type { ComponentLogger, Metric, Metrics, PeerId, Startable } from '@libp2p/interface' +import type { ComponentLogger, Metric, Metrics, PeerId, RoutingOptions, Startable } from '@libp2p/interface' import type { DeferredPromise } from 'p-defer' export interface CleanUpEvents { @@ -19,7 +19,7 @@ export interface CleanUpEvents { } export interface QueryManagerInit { - lan?: boolean + logPrefix: string disjointPaths?: number alpha?: number initialQuerySelfHasRun: DeferredPromise @@ -32,8 +32,12 @@ export interface QueryManagerComponents { logger: ComponentLogger } -export interface QueryOptions extends RootQueryOptions { +export interface QueryOptions extends RoutingOptions { + /** + * A timeout for subqueries executed as part of the main query + */ queryFuncTimeout?: number + isSelfQuery?: boolean } @@ -41,7 +45,6 @@ export interface QueryOptions extends RootQueryOptions { * Keeps track of all running queries */ export class QueryManager implements Startable { - private readonly lan: boolean public disjointPaths: number private readonly alpha: number private readonly shutDownController: AbortController @@ -51,18 +54,19 @@ export class QueryManager implements Startable { private readonly peerId: PeerId private readonly routingTable: RoutingTable private initialQuerySelfHasRun?: DeferredPromise + private readonly logPrefix: string private readonly metrics?: { runningQueries: Metric queryTime: Metric } constructor (components: QueryManagerComponents, init: QueryManagerInit) { - const { lan = false, disjointPaths = K, alpha = ALPHA } = init + const { disjointPaths = K, alpha = ALPHA, logPrefix } = init + this.logPrefix = logPrefix this.disjointPaths = disjointPaths ?? K this.running = false this.alpha = alpha ?? ALPHA - this.lan = lan this.queries = 0 this.initialQuerySelfHasRun = init.initialQuerySelfHasRun this.routingTable = init.routingTable @@ -71,8 +75,8 @@ export class QueryManager implements Startable { if (components.metrics != null) { this.metrics = { - runningQueries: components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_running_queries`), - queryTime: components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_query_time_seconds`) + runningQueries: components.metrics.registerMetric(`${logPrefix.replaceAll(':', '_')}_running_queries`), + queryTime: components.metrics.registerMetric(`${logPrefix.replaceAll(':', '_')}_query_time_seconds`) } } @@ -129,7 +133,7 @@ export class QueryManager implements Startable { // so make sure we don't make a lot of noise in the logs setMaxListeners(Infinity, signal) - const log = this.logger.forComponent(`libp2p:kad-dht:${this.lan ? 'lan' : 'wan'}:query:` + uint8ArrayToString(key, 'base58btc')) + const log = this.logger.forComponent(`${this.logPrefix}:query:` + uint8ArrayToString(key, 'base58btc')) // query a subset of peers up to `kBucketSize / 2` in length const startTime = Date.now() diff --git a/packages/kad-dht/src/query/query-path.ts b/packages/kad-dht/src/query/query-path.ts index 79c2285992..05e0df23f6 100644 --- a/packages/kad-dht/src/query/query-path.ts +++ b/packages/kad-dht/src/query/query-path.ts @@ -6,14 +6,14 @@ import { convertPeerId, convertBuffer } from '../utils.js' import { queryErrorEvent } from './events.js' import { queueToGenerator } from './utils.js' import type { CleanUpEvents } from './manager.js' -import type { QueryEvent, QueryOptions } from '../index.js' +import type { QueryEvent } from '../index.js' import type { QueryFunc } from '../query/types.js' -import type { Logger, TypedEventTarget, PeerId } from '@libp2p/interface' +import type { Logger, TypedEventTarget, PeerId, RoutingOptions } from '@libp2p/interface' import type { PeerSet } from '@libp2p/peer-collections' const MAX_XOR = BigInt('0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF') -export interface QueryPathOptions extends QueryOptions { +export interface QueryPathOptions extends RoutingOptions { /** * What are we trying to find */ diff --git a/packages/kad-dht/src/routing-table/index.ts b/packages/kad-dht/src/routing-table/index.ts index adcdf7d3ce..106bafc47a 100644 --- a/packages/kad-dht/src/routing-table/index.ts +++ b/packages/kad-dht/src/routing-table/index.ts @@ -1,9 +1,11 @@ -import { TypedEventEmitter } from '@libp2p/interface' +import { CodeError, TypedEventEmitter } from '@libp2p/interface' import { PeerSet } from '@libp2p/peer-collections' -import Queue from 'p-queue' +import { PeerJobQueue } from '@libp2p/utils/peer-job-queue' +import { pbStream } from 'it-protobuf-stream' +import { Message, MessageType } from '../message/dht.js' import * as utils from '../utils.js' import { KBucket, type PingEventDetails } from './k-bucket.js' -import type { ComponentLogger, Logger, Metric, Metrics, PeerId, PeerStore, Startable } from '@libp2p/interface' +import type { ComponentLogger, Logger, Metric, Metrics, PeerId, PeerStore, Startable, Stream } from '@libp2p/interface' import type { ConnectionManager } from '@libp2p/interface-internal' export const KAD_CLOSE_TAG_NAME = 'kad-close' @@ -13,7 +15,7 @@ export const PING_TIMEOUT = 10000 export const PING_CONCURRENCY = 10 export interface RoutingTableInit { - lan: boolean + logPrefix: string protocol: string kBucketSize?: number pingTimeout?: number @@ -42,18 +44,17 @@ export interface RoutingTableEvents { export class RoutingTable extends TypedEventEmitter implements Startable { public kBucketSize: number public kb?: KBucket - public pingQueue: Queue + public pingQueue: PeerJobQueue private readonly log: Logger private readonly components: RoutingTableComponents - private readonly lan: boolean private readonly pingTimeout: number private readonly pingConcurrency: number private running: boolean private readonly protocol: string private readonly tagName: string private readonly tagValue: number - private metrics?: { + private readonly metrics?: { routingTableSize: Metric pingQueueSize: Metric pingRunning: Metric @@ -62,14 +63,13 @@ export class RoutingTable extends TypedEventEmitter implemen constructor (components: RoutingTableComponents, init: RoutingTableInit) { super() - const { kBucketSize, pingTimeout, lan, pingConcurrency, protocol, tagName, tagValue } = init + const { kBucketSize, pingTimeout, logPrefix, pingConcurrency, protocol, tagName, tagValue } = init this.components = components - this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:routing-table`) + this.log = components.logger.forComponent(`${logPrefix}:routing-table`) this.kBucketSize = kBucketSize ?? KBUCKET_SIZE this.pingTimeout = pingTimeout ?? PING_TIMEOUT this.pingConcurrency = pingConcurrency ?? PING_CONCURRENCY - this.lan = lan this.running = false this.protocol = protocol this.tagName = tagName ?? KAD_CLOSE_TAG_NAME @@ -80,11 +80,20 @@ export class RoutingTable extends TypedEventEmitter implemen this.metrics?.pingRunning.update(this.pingQueue.pending) } - this.pingQueue = new Queue({ concurrency: this.pingConcurrency }) + this.pingQueue = new PeerJobQueue({ concurrency: this.pingConcurrency }) this.pingQueue.addListener('add', updatePingQueueSizeMetric) this.pingQueue.addListener('next', updatePingQueueSizeMetric) + this.pingQueue.addListener('error', err => { + this.log.error('error pinging peer', err) + }) - this._onPing = this._onPing.bind(this) + if (this.components.metrics != null) { + this.metrics = { + routingTableSize: this.components.metrics.registerMetric(`${logPrefix.replaceAll(':', '_')}_routing_table_size`), + pingQueueSize: this.components.metrics.registerMetric(`${logPrefix.replaceAll(':', '_')}_ping_queue_size`), + pingRunning: this.components.metrics.registerMetric(`${logPrefix.replaceAll(':', '_')}_ping_running`) + } + } } isStarted (): boolean { @@ -94,14 +103,6 @@ export class RoutingTable extends TypedEventEmitter implemen async start (): Promise { this.running = true - if (this.components.metrics != null) { - this.metrics = { - routingTableSize: this.components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_routing_table_size`), - pingQueueSize: this.components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_ping_queue_size`), - pingRunning: this.components.metrics.registerMetric(`libp2p_kad_dht_${this.lan ? 'lan' : 'wan'}_ping_running`) - } - } - const kBuck = new KBucket({ localNodeId: await utils.convertPeerId(this.components.peerId), numberOfNodesPerKBucket: this.kBucketSize, @@ -110,7 +111,11 @@ export class RoutingTable extends TypedEventEmitter implemen this.kb = kBuck // test whether to evict peers - kBuck.addEventListener('ping', this._onPing) + kBuck.addEventListener('ping', (evt) => { + this._onPing(evt).catch(err => { + this.log.error('could not process k-bucket ping event', err) + }) + }) // tag kad-close peers this._tagPeers(kBuck) @@ -186,60 +191,77 @@ export class RoutingTable extends TypedEventEmitter implemen * `oldContacts` will not be empty and is the list of contacts that * have not been contacted for the longest. */ - _onPing (evt: CustomEvent): void { + async _onPing (evt: CustomEvent): Promise { + if (!this.running) { + return + } + const { oldContacts, newContact } = evt.detail - // add to a queue so multiple ping requests do not overlap and we don't - // flood the network with ping requests if lots of newContact requests - // are received - this.pingQueue.add(async () => { - if (!this.running) { - return - } + const results = await Promise.all( + oldContacts.map(async oldContact => { + // if a previous ping wants us to ping this contact, re-use the result + if (this.pingQueue.hasJob(oldContact.peer)) { + return this.pingQueue.joinJob(oldContact.peer) + } - let responded = 0 + return this.pingQueue.add(async () => { + let stream: Stream | undefined - try { - await Promise.all( - oldContacts.map(async oldContact => { - try { - const options = { - signal: AbortSignal.timeout(this.pingTimeout) - } + try { + const options = { + signal: AbortSignal.timeout(this.pingTimeout) + } - this.log('pinging old contact %p', oldContact.peer) - const connection = await this.components.connectionManager.openConnection(oldContact.peer, options) - const stream = await connection.newStream(this.protocol, options) - await stream.close() - responded++ - } catch (err: any) { - if (this.running && this.kb != null) { - // only evict peers if we are still running, otherwise we evict when dialing is - // cancelled due to shutdown in progress - this.log.error('could not ping peer %p', oldContact.peer, err) - this.log('evicting old contact after ping failed %p', oldContact.peer) - this.kb.remove(oldContact.id) - } - } finally { - this.metrics?.routingTableSize.update(this.size) + this.log('pinging old contact %p', oldContact.peer) + const connection = await this.components.connectionManager.openConnection(oldContact.peer, options) + stream = await connection.newStream(this.protocol, options) + + const pb = pbStream(stream) + await pb.write({ + type: MessageType.PING + }, Message, options) + const response = await pb.read(Message, options) + + await pb.unwrap().close() + + if (response.type !== MessageType.PING) { + throw new CodeError(`Incorrect message type received, expected PING got ${response.type}`, 'ERR_BAD_PING_RESPONSE') } - }) - ) - if (this.running && responded < oldContacts.length && this.kb != null) { - this.log('adding new contact %p', newContact.peer) - this.kb.add(newContact) - } - } catch (err: any) { - this.log.error('could not process k-bucket ping event', err) - } - }) - .catch(err => { - this.log.error('could not process k-bucket ping event', err) + return true + } catch (err: any) { + if (this.running && this.kb != null) { + // only evict peers if we are still running, otherwise we evict + // when dialing is cancelled due to shutdown in progress + this.log.error('could not ping peer %p', oldContact.peer, err) + this.log('evicting old contact after ping failed %p', oldContact.peer) + this.kb.remove(oldContact.id) + } + + stream?.abort(err) + + return false + } finally { + this.metrics?.routingTableSize.update(this.size) + } + }, { + peerId: oldContact.peer + }) }) + ) + + const responded = results + .filter(res => res) + .length + + if (this.running && responded < oldContacts.length && this.kb != null) { + this.log('adding new contact %p', newContact.peer) + this.kb.add(newContact) + } } // -- Public Interface diff --git a/packages/kad-dht/src/routing-table/refresh.ts b/packages/kad-dht/src/routing-table/refresh.ts index 6a2c183886..d5cbd87baa 100644 --- a/packages/kad-dht/src/routing-table/refresh.ts +++ b/packages/kad-dht/src/routing-table/refresh.ts @@ -21,7 +21,7 @@ export interface RoutingTableRefreshComponents { export interface RoutingTableRefreshInit { peerRouting: PeerRouting routingTable: RoutingTable - lan: boolean + logPrefix: string refreshInterval?: number refreshQueryTimeout?: number } @@ -40,8 +40,8 @@ export class RoutingTableRefresh { private refreshTimeoutId?: ReturnType constructor (components: RoutingTableRefreshComponents, init: RoutingTableRefreshInit) { - const { peerRouting, routingTable, refreshInterval, refreshQueryTimeout, lan } = init - this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:routing-table:refresh`) + const { peerRouting, routingTable, refreshInterval, refreshQueryTimeout, logPrefix } = init + this.log = components.logger.forComponent(`${logPrefix}:routing-table:refresh`) this.peerRouting = peerRouting this.routingTable = routingTable this.refreshInterval = refreshInterval ?? TABLE_REFRESH_INTERVAL @@ -147,7 +147,7 @@ export class RoutingTableRefresh { maxCommonPrefix = MAX_COMMON_PREFIX_LENGTH } - const dates = [] + const dates: Date[] = [] for (let i = 0; i <= maxCommonPrefix; i++) { // defaults to the zero value if we haven't refreshed it yet. diff --git a/packages/kad-dht/src/rpc/handlers/add-provider.ts b/packages/kad-dht/src/rpc/handlers/add-provider.ts index f2dd3a3c43..0d15f3cf2b 100644 --- a/packages/kad-dht/src/rpc/handlers/add-provider.ts +++ b/packages/kad-dht/src/rpc/handlers/add-provider.ts @@ -1,6 +1,8 @@ import { CodeError } from '@libp2p/interface' +import { peerIdFromBytes } from '@libp2p/peer-id' +import { multiaddr } from '@multiformats/multiaddr' import { CID } from 'multiformats/cid' -import type { Message } from '../../message/index.js' +import type { Message } from '../../message/dht.js' import type { Providers } from '../../providers' import type { DHTMessageHandler } from '../index.js' import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface' @@ -11,6 +13,7 @@ export interface AddProviderComponents { export interface AddProviderHandlerInit { providers: Providers + logPrefix: string } export class AddProviderHandler implements DHTMessageHandler { @@ -18,7 +21,7 @@ export class AddProviderHandler implements DHTMessageHandler { private readonly log: Logger constructor (components: AddProviderComponents, init: AddProviderHandlerInit) { - this.log = components.logger.forComponent('libp2p:kad-dht:rpc:handlers:add-provider') + this.log = components.logger.forComponent(`${init.logPrefix}:rpc:handlers:add-provider`) this.providers = init.providers } @@ -37,14 +40,14 @@ export class AddProviderHandler implements DHTMessageHandler { throw new CodeError('Invalid CID', 'ERR_INVALID_CID') } - if (msg.providerPeers == null || msg.providerPeers.length === 0) { + if (msg.providers == null || msg.providers.length === 0) { this.log.error('no providers found in message') } await Promise.all( - msg.providerPeers.map(async (pi) => { + msg.providers.map(async (pi) => { // Ignore providers not from the originator - if (!pi.id.equals(peerId)) { + if (!peerId.equals(pi.id)) { this.log('invalid provider peer %p from %p', pi.id, peerId) return } @@ -54,9 +57,9 @@ export class AddProviderHandler implements DHTMessageHandler { return } - this.log('received provider %p for %s (addrs %s)', peerId, cid, pi.multiaddrs.map((m) => m.toString())) + this.log('received provider %p for %s (addrs %s)', peerId, cid, pi.multiaddrs.map((m) => multiaddr(m).toString())) - await this.providers.addProvider(cid, pi.id) + await this.providers.addProvider(cid, peerIdFromBytes(pi.id)) }) ) diff --git a/packages/kad-dht/src/rpc/handlers/find-node.ts b/packages/kad-dht/src/rpc/handlers/find-node.ts index a18713d9c8..162a8ce775 100644 --- a/packages/kad-dht/src/rpc/handlers/find-node.ts +++ b/packages/kad-dht/src/rpc/handlers/find-node.ts @@ -1,10 +1,9 @@ +import { CodeError } from '@libp2p/interface' import { protocols } from '@multiformats/multiaddr' import { equals as uint8ArrayEquals } from 'uint8arrays' -import { Message } from '../../message/index.js' -import { - removePrivateAddresses, - removePublicAddresses -} from '../../utils.js' +import { MessageType } from '../../message/dht.js' +import type { PeerInfoMapper } from '../../index.js' +import type { Message } from '../../message/dht.js' import type { PeerRouting } from '../../peer-routing/index.js' import type { DHTMessageHandler } from '../index.js' import type { ComponentLogger, Logger, PeerId, PeerInfo } from '@libp2p/interface' @@ -12,7 +11,8 @@ import type { AddressManager } from '@libp2p/interface-internal' export interface FindNodeHandlerInit { peerRouting: PeerRouting - lan: boolean + logPrefix: string + peerInfoMapper: PeerInfoMapper } export interface FindNodeHandlerComponents { @@ -23,19 +23,19 @@ export interface FindNodeHandlerComponents { export class FindNodeHandler implements DHTMessageHandler { private readonly peerRouting: PeerRouting - private readonly lan: boolean + private readonly peerInfoMapper: PeerInfoMapper private readonly peerId: PeerId private readonly addressManager: AddressManager private readonly log: Logger constructor (components: FindNodeHandlerComponents, init: FindNodeHandlerInit) { - const { peerRouting, lan } = init + const { peerRouting, logPrefix } = init - this.log = components.logger.forComponent('libp2p:kad-dht:rpc:handlers:find-node') + this.log = components.logger.forComponent(`${logPrefix}:rpc:handlers:find-node`) this.peerId = components.peerId this.addressManager = components.addressManager this.peerRouting = peerRouting - this.lan = Boolean(lan) + this.peerInfoMapper = init.peerInfoMapper } /** @@ -46,6 +46,10 @@ export class FindNodeHandler implements DHTMessageHandler { let closer: PeerInfo[] = [] + if (msg.key == null) { + throw new CodeError('Invalid FIND_NODE message received - key was missing', 'ERR_INVALID_MESSAGE') + } + if (uint8ArrayEquals(this.peerId.toBytes(), msg.key)) { closer = [{ id: this.peerId, @@ -55,15 +59,20 @@ export class FindNodeHandler implements DHTMessageHandler { closer = await this.peerRouting.getCloserPeersOffline(msg.key, peerId) } - closer = closer - .map(this.lan ? removePublicAddresses : removePrivateAddresses) - .filter(({ multiaddrs }) => multiaddrs.length) - - const response = new Message(msg.type, new Uint8Array(0), msg.clusterLevel) + const response: Message = { + type: MessageType.FIND_NODE, + clusterLevel: msg.clusterLevel, + closer: closer + .map(this.peerInfoMapper) + .filter(({ multiaddrs }) => multiaddrs.length) + .map(peerInfo => ({ + id: peerInfo.id.toBytes(), + multiaddrs: peerInfo.multiaddrs.map(ma => ma.bytes) + })), + providers: [] + } - if (closer.length > 0) { - response.closerPeers = closer - } else { + if (response.closer.length === 0) { this.log('could not find any peers closer to %b than %p', msg.key, peerId) } diff --git a/packages/kad-dht/src/rpc/handlers/get-providers.ts b/packages/kad-dht/src/rpc/handlers/get-providers.ts index 96cf040f1c..ef13fa5353 100644 --- a/packages/kad-dht/src/rpc/handlers/get-providers.ts +++ b/packages/kad-dht/src/rpc/handlers/get-providers.ts @@ -1,10 +1,8 @@ import { CodeError } from '@libp2p/interface' import { CID } from 'multiformats/cid' -import { Message } from '../../message/index.js' -import { - removePrivateAddresses, - removePublicAddresses -} from '../../utils.js' +import { MessageType } from '../../message/dht.js' +import type { PeerInfoMapper } from '../../index.js' +import type { Message } from '../../message/dht.js' import type { PeerRouting } from '../../peer-routing/index.js' import type { Providers } from '../../providers.js' import type { DHTMessageHandler } from '../index.js' @@ -14,7 +12,8 @@ import type { Multiaddr } from '@multiformats/multiaddr' export interface GetProvidersHandlerInit { peerRouting: PeerRouting providers: Providers - lan: boolean + logPrefix: string + peerInfoMapper: PeerInfoMapper } export interface GetProvidersHandlerComponents { @@ -25,21 +24,25 @@ export interface GetProvidersHandlerComponents { export class GetProvidersHandler implements DHTMessageHandler { private readonly peerRouting: PeerRouting private readonly providers: Providers - private readonly lan: boolean private readonly peerStore: PeerStore + private readonly peerInfoMapper: PeerInfoMapper private readonly log: Logger constructor (components: GetProvidersHandlerComponents, init: GetProvidersHandlerInit) { - const { peerRouting, providers, lan } = init + const { peerRouting, providers, logPrefix } = init - this.log = components.logger.forComponent('libp2p:kad-dht:rpc:handlers:get-providers') + this.log = components.logger.forComponent(`${logPrefix}:rpc:handlers:get-providers`) this.peerStore = components.peerStore this.peerRouting = peerRouting this.providers = providers - this.lan = Boolean(lan) + this.peerInfoMapper = init.peerInfoMapper } async handle (peerId: PeerId, msg: Message): Promise { + if (msg.key == null) { + throw new CodeError('Invalid FIND_NODE message received - key was missing', 'ERR_INVALID_MESSAGE') + } + let cid try { cid = CID.decode(msg.key) @@ -56,17 +59,28 @@ export class GetProvidersHandler implements DHTMessageHandler { const providerPeers = await this._getPeers(peers) const closerPeers = await this._getPeers(closer.map(({ id }) => id)) - const response = new Message(msg.type, msg.key, msg.clusterLevel) - - if (providerPeers.length > 0) { - response.providerPeers = providerPeers + const response: Message = { + type: MessageType.GET_PROVIDERS, + key: msg.key, + clusterLevel: msg.clusterLevel, + closer: closerPeers + .map(this.peerInfoMapper) + .filter(({ multiaddrs }) => multiaddrs.length) + .map(peerInfo => ({ + id: peerInfo.id.toBytes(), + multiaddrs: peerInfo.multiaddrs.map(ma => ma.bytes) + })), + providers: providerPeers + .map(this.peerInfoMapper) + .filter(({ multiaddrs }) => multiaddrs.length) + .map(peerInfo => ({ + id: peerInfo.id.toBytes(), + multiaddrs: peerInfo.multiaddrs.map(ma => ma.bytes) + })) } - if (closerPeers.length > 0) { - response.closerPeers = closerPeers - } + this.log('got %s providers %s closerPeers', response.providers.length, response.closer.length) - this.log('got %s providers %s closerPeers', providerPeers.length, closerPeers.length) return response } @@ -76,13 +90,12 @@ export class GetProvidersHandler implements DHTMessageHandler { async _getPeers (peerIds: PeerId[]): Promise { const output: PeerInfo[] = [] - const addrFilter = this.lan ? removePublicAddresses : removePrivateAddresses for (const peerId of peerIds) { try { const peer = await this.peerStore.get(peerId) - const peerAfterFilter = addrFilter({ + const peerAfterFilter = this.peerInfoMapper({ id: peerId, multiaddrs: peer.addresses.map(({ multiaddr }) => multiaddr) }) diff --git a/packages/kad-dht/src/rpc/handlers/get-value.ts b/packages/kad-dht/src/rpc/handlers/get-value.ts index 50475bf191..8ef163bbd5 100644 --- a/packages/kad-dht/src/rpc/handlers/get-value.ts +++ b/packages/kad-dht/src/rpc/handlers/get-value.ts @@ -2,9 +2,10 @@ import { CodeError } from '@libp2p/interface' import { MAX_RECORD_AGE } from '../../constants.js' -import { Message, MESSAGE_TYPE } from '../../message/index.js' +import { MessageType } from '../../message/dht.js' import { Libp2pRecord } from '../../record/index.js' import { bufferToRecordKey, isPublicKeyKey, fromPublicKeyKey } from '../../utils.js' +import type { Message } from '../../message/dht.js' import type { PeerRouting } from '../../peer-routing/index.js' import type { DHTMessageHandler } from '../index.js' import type { ComponentLogger, Logger, PeerId, PeerStore } from '@libp2p/interface' @@ -12,6 +13,7 @@ import type { Datastore } from 'interface-datastore' export interface GetValueHandlerInit { peerRouting: PeerRouting + logPrefix: string } export interface GetValueHandlerComponents { @@ -27,7 +29,7 @@ export class GetValueHandler implements DHTMessageHandler { private readonly log: Logger constructor (components: GetValueHandlerComponents, init: GetValueHandlerInit) { - this.log = components.logger.forComponent('libp2p:kad-dht:rpc:handlers:get-value') + this.log = components.logger.forComponent(`${init.logPrefix}:rpc:handlers:get-value`) this.peerStore = components.peerStore this.datastore = components.datastore this.peerRouting = init.peerRouting @@ -42,7 +44,13 @@ export class GetValueHandler implements DHTMessageHandler { throw new CodeError('Invalid key', 'ERR_INVALID_KEY') } - const response = new Message(MESSAGE_TYPE.GET_VALUE, key, msg.clusterLevel) + const response: Message = { + type: MessageType.GET_VALUE, + key, + clusterLevel: msg.clusterLevel, + closer: [], + providers: [] + } if (isPublicKeyKey(key)) { this.log('is public key') @@ -65,24 +73,27 @@ export class GetValueHandler implements DHTMessageHandler { if (pubKey != null) { this.log('returning found public key') - response.record = new Libp2pRecord(key, pubKey, new Date()) + response.record = new Libp2pRecord(key, pubKey, new Date()).serialize() return response } } const [record, closer] = await Promise.all([ this._checkLocalDatastore(key), - this.peerRouting.getCloserPeersOffline(msg.key, peerId) + this.peerRouting.getCloserPeersOffline(key, peerId) ]) if (record != null) { this.log('had record for %b in local datastore', key) - response.record = record + response.record = record.serialize() } if (closer.length > 0) { this.log('had %s closer peers in routing table', closer.length) - response.closerPeers = closer + response.closer = closer.map(peerInfo => ({ + id: peerInfo.id.toBytes(), + multiaddrs: peerInfo.multiaddrs.map(ma => ma.bytes) + })) } return response diff --git a/packages/kad-dht/src/rpc/handlers/ping.ts b/packages/kad-dht/src/rpc/handlers/ping.ts index 1a63aad153..f1e16357c7 100644 --- a/packages/kad-dht/src/rpc/handlers/ping.ts +++ b/packages/kad-dht/src/rpc/handlers/ping.ts @@ -1,4 +1,4 @@ -import type { Message } from '../../message/index.js' +import type { Message } from '../../message/dht.js' import type { DHTMessageHandler } from '../index.js' import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface' @@ -6,11 +6,15 @@ export interface PingComponents { logger: ComponentLogger } +export interface PingHandlerInit { + logPrefix: string +} + export class PingHandler implements DHTMessageHandler { private readonly log: Logger - constructor (components: PingComponents) { - this.log = components.logger.forComponent('libp2p:kad-dht:rpc:handlers:ping') + constructor (components: PingComponents, init: PingHandlerInit) { + this.log = components.logger.forComponent(`${init.logPrefix}:rpc:handlers:ping`) } async handle (peerId: PeerId, msg: Message): Promise { diff --git a/packages/kad-dht/src/rpc/handlers/put-value.ts b/packages/kad-dht/src/rpc/handlers/put-value.ts index 9fe1fe76e4..46a401c8cd 100644 --- a/packages/kad-dht/src/rpc/handlers/put-value.ts +++ b/packages/kad-dht/src/rpc/handlers/put-value.ts @@ -1,14 +1,16 @@ import { CodeError } from '@libp2p/interface' +import { Libp2pRecord } from '../../record/index.js' import { verifyRecord } from '../../record/validators.js' import { bufferToRecordKey } from '../../utils.js' import type { Validators } from '../../index.js' -import type { Message } from '../../message/index.js' +import type { Message } from '../../message/dht.js' import type { DHTMessageHandler } from '../index.js' import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' export interface PutValueHandlerInit { validators: Validators + logPrefix: string } export interface PutValueHandlerComponents { @@ -25,7 +27,7 @@ export class PutValueHandler implements DHTMessageHandler { const { validators } = init this.components = components - this.log = components.logger.forComponent('libp2p:kad-dht:rpc:handlers:put-value') + this.log = components.logger.forComponent(`${init.logPrefix}:rpc:handlers:put-value`) this.validators = validators } @@ -33,9 +35,7 @@ export class PutValueHandler implements DHTMessageHandler { const key = msg.key this.log('%p asked us to store value for key %b', peerId, key) - const record = msg.record - - if (record == null) { + if (msg.record == null) { const errMsg = `Empty record from: ${peerId.toString()}` this.log.error(errMsg) @@ -43,11 +43,13 @@ export class PutValueHandler implements DHTMessageHandler { } try { - await verifyRecord(this.validators, record) + const deserializedRecord = Libp2pRecord.deserialize(msg.record) + + await verifyRecord(this.validators, deserializedRecord) - record.timeReceived = new Date() - const recordKey = bufferToRecordKey(record.key) - await this.components.datastore.put(recordKey, record.serialize().subarray()) + deserializedRecord.timeReceived = new Date() + const recordKey = bufferToRecordKey(deserializedRecord.key) + await this.components.datastore.put(recordKey, deserializedRecord.serialize().subarray()) this.log('put record for %b into datastore under key %k', key, recordKey) } catch (err: any) { this.log('did not put record for key %b into datastore %o', key, err) diff --git a/packages/kad-dht/src/rpc/index.ts b/packages/kad-dht/src/rpc/index.ts index 8a77a1730b..4f5c85a160 100644 --- a/packages/kad-dht/src/rpc/index.ts +++ b/packages/kad-dht/src/rpc/index.ts @@ -1,13 +1,13 @@ import * as lp from 'it-length-prefixed' import { pipe } from 'it-pipe' -import { Message, MESSAGE_TYPE } from '../message/index.js' +import { Message, MessageType } from '../message/dht.js' import { AddProviderHandler } from './handlers/add-provider.js' import { FindNodeHandler, type FindNodeHandlerComponents } from './handlers/find-node.js' import { GetProvidersHandler, type GetProvidersHandlerComponents } from './handlers/get-providers.js' import { GetValueHandler, type GetValueHandlerComponents } from './handlers/get-value.js' import { PingHandler } from './handlers/ping.js' import { PutValueHandler, type PutValueHandlerComponents } from './handlers/put-value.js' -import type { Validators } from '../index.js' +import type { PeerInfoMapper, Validators } from '../index.js' import type { PeerRouting } from '../peer-routing' import type { Providers } from '../providers' import type { RoutingTable } from '../routing-table' @@ -23,7 +23,8 @@ export interface RPCInit { providers: Providers peerRouting: PeerRouting validators: Validators - lan: boolean + logPrefix: string + peerInfoMapper: PeerInfoMapper } export interface RPCComponents extends GetValueHandlerComponents, PutValueHandlerComponents, FindNodeHandlerComponents, GetProvidersHandlerComponents { @@ -36,17 +37,17 @@ export class RPC { private readonly log: Logger constructor (components: RPCComponents, init: RPCInit) { - const { providers, peerRouting, validators, lan } = init + const { providers, peerRouting, validators, logPrefix, peerInfoMapper } = init - this.log = components.logger.forComponent('libp2p:kad-dht:rpc') + this.log = components.logger.forComponent(`${logPrefix}:rpc`) this.routingTable = init.routingTable this.handlers = { - [MESSAGE_TYPE.GET_VALUE]: new GetValueHandler(components, { peerRouting }), - [MESSAGE_TYPE.PUT_VALUE]: new PutValueHandler(components, { validators }), - [MESSAGE_TYPE.FIND_NODE]: new FindNodeHandler(components, { peerRouting, lan }), - [MESSAGE_TYPE.ADD_PROVIDER]: new AddProviderHandler(components, { providers }), - [MESSAGE_TYPE.GET_PROVIDERS]: new GetProvidersHandler(components, { peerRouting, providers, lan }), - [MESSAGE_TYPE.PING]: new PingHandler(components) + [MessageType.GET_VALUE.toString()]: new GetValueHandler(components, { peerRouting, logPrefix }), + [MessageType.PUT_VALUE.toString()]: new PutValueHandler(components, { validators, logPrefix }), + [MessageType.FIND_NODE.toString()]: new FindNodeHandler(components, { peerRouting, logPrefix, peerInfoMapper }), + [MessageType.ADD_PROVIDER.toString()]: new AddProviderHandler(components, { providers, logPrefix }), + [MessageType.GET_PROVIDERS.toString()]: new GetProvidersHandler(components, { peerRouting, providers, logPrefix, peerInfoMapper }), + [MessageType.PING.toString()]: new PingHandler(components, { logPrefix }) } } @@ -93,13 +94,13 @@ export class RPC { async function * (source) { for await (const msg of source) { // handle the message - const desMessage = Message.deserialize(msg) + const desMessage = Message.decode(msg) self.log('incoming %s from %p', desMessage.type, peerId) const res = await self.handleMessage(peerId, desMessage) // Not all handlers will return a response if (res != null) { - yield res.serialize() + yield Message.encode(res) } } }, diff --git a/packages/kad-dht/src/topology-listener.ts b/packages/kad-dht/src/topology-listener.ts index cab72514b1..2b24b3417b 100644 --- a/packages/kad-dht/src/topology-listener.ts +++ b/packages/kad-dht/src/topology-listener.ts @@ -4,7 +4,7 @@ import type { Logger, PeerId, Startable } from '@libp2p/interface' export interface TopologyListenerInit { protocol: string - lan: boolean + logPrefix: string } export interface TopologyListenerEvents { @@ -24,10 +24,10 @@ export class TopologyListener extends TypedEventEmitter constructor (components: KadDHTComponents, init: TopologyListenerInit) { super() - const { protocol, lan } = init + const { protocol, logPrefix } = init this.components = components - this.log = components.logger.forComponent(`libp2p:kad-dht:topology-listener:${lan ? 'lan' : 'wan'}`) + this.log = components.logger.forComponent(`${logPrefix}:topology-listener`) this.running = false this.protocol = protocol } diff --git a/packages/kad-dht/src/utils.ts b/packages/kad-dht/src/utils.ts index 583c552253..6bd44b2c5e 100644 --- a/packages/kad-dht/src/utils.ts +++ b/packages/kad-dht/src/utils.ts @@ -8,11 +8,12 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { RECORD_KEY_PREFIX } from './constants.js' import { Libp2pRecord } from './record/index.js' import type { PeerId, PeerInfo } from '@libp2p/interface' +import type { Multiaddr } from '@multiformats/multiaddr' // const IPNS_PREFIX = uint8ArrayFromString('/ipns/') const PK_PREFIX = uint8ArrayFromString('/pk/') -export function removePrivateAddresses (peer: PeerInfo): PeerInfo { +export function removePrivateAddressesMapper (peer: PeerInfo): PeerInfo { return { ...peer, multiaddrs: peer.multiaddrs.filter(multiaddr => { @@ -48,7 +49,7 @@ export function removePrivateAddresses (peer: PeerInfo): PeerInfo { } } -export function removePublicAddresses (peer: PeerInfo): PeerInfo { +export function removePublicAddressesMapper (peer: PeerInfo): PeerInfo { return { ...peer, multiaddrs: peer.multiaddrs.filter(multiaddr => { @@ -78,6 +79,10 @@ export function removePublicAddresses (peer: PeerInfo): PeerInfo { } } +export function passthroughMapper (info: PeerInfo): PeerInfo { + return info +} + /** * Creates a DHT ID by hashing a given Uint8Array */ @@ -148,3 +153,37 @@ export function debounce (callback: () => void, wait: number = 100): () => void timeout = setTimeout(() => { callback() }, wait) } } + +// see https://github.com/multiformats/multiaddr/blob/master/protocols.csv +const P2P_CIRCUIT_CODE = 290 +const DNS4_CODE = 54 +const DNS6_CODE = 55 +const DNSADDR_CODE = 56 +const IP4_CODE = 4 +const IP6_CODE = 41 + +export function multiaddrIsPublic (multiaddr: Multiaddr): boolean { + const tuples = multiaddr.stringTuples() + + // p2p-circuit should not enable server mode + for (const tuple of tuples) { + if (tuple[0] === P2P_CIRCUIT_CODE) { + return false + } + } + + // dns4 or dns6 or dnsaddr + if (tuples[0][0] === DNS4_CODE || tuples[0][0] === DNS6_CODE || tuples[0][0] === DNSADDR_CODE) { + return true + } + + // ip4 or ip6 + if (tuples[0][0] === IP4_CODE || tuples[0][0] === IP6_CODE) { + const result = isPrivateIp(`${tuples[0][1]}`) + const isPublic = result == null || !result + + return isPublic + } + + return false +} diff --git a/packages/kad-dht/test/enable-server-mode.spec.ts b/packages/kad-dht/test/enable-server-mode.spec.ts index 80a93abdc1..1497b52499 100644 --- a/packages/kad-dht/test/enable-server-mode.spec.ts +++ b/packages/kad-dht/test/enable-server-mode.spec.ts @@ -30,9 +30,9 @@ describe('enable server mode', () => { testCases.forEach(([name, addr, result]) => { it(name, async function () { - const dht = await tdht.spawn() + const dht = await tdht.spawn({ clientMode: undefined }) - await expect(dht.getMode()).to.eventually.equal('client') + expect(dht.getMode()).to.equal('client') dht.components.events.safeDispatchEvent('self:peer:update', { detail: { @@ -53,7 +53,7 @@ describe('enable server mode', () => { await delay(100) - await expect(dht.getMode()).to.eventually.equal(result, `did not change to "${result}" mode after updating with address ${addr}`) + expect(dht.getMode()).to.equal(result, `did not change to "${result}" mode after updating with address ${addr}`) dht.components.events.safeDispatchEvent('self:peer:update', { detail: { @@ -68,7 +68,7 @@ describe('enable server mode', () => { await delay(100) - await expect(dht.getMode()).to.eventually.equal('client', `did not reset to client mode after updating with address ${addr}`) + expect(dht.getMode()).to.equal('client', `did not reset to client mode after updating with address ${addr}`) }) }) }) diff --git a/packages/kad-dht/test/fixtures/match-peer-id.ts b/packages/kad-dht/test/fixtures/match-peer-id.ts new file mode 100644 index 0000000000..ad9dae226b --- /dev/null +++ b/packages/kad-dht/test/fixtures/match-peer-id.ts @@ -0,0 +1,6 @@ +import Sinon from 'sinon' +import type { PeerId } from '@libp2p/interface' + +export function matchPeerId (peerId: PeerId): Sinon.SinonMatcher { + return Sinon.match(p => p.toString() === peerId.toString()) +} diff --git a/packages/kad-dht/test/generate-peers/generate-peers.node.ts b/packages/kad-dht/test/generate-peers/generate-peers.node.ts index 5f482ae3ac..a24875796c 100644 --- a/packages/kad-dht/test/generate-peers/generate-peers.node.ts +++ b/packages/kad-dht/test/generate-peers/generate-peers.node.ts @@ -61,7 +61,7 @@ describe.skip('generate peers', function () { } const table = new RoutingTable(components, { kBucketSize: 20, - lan: false, + logPrefix: '', protocol: '/ipfs/kad/1.0.0' }) refresh = new RoutingTableRefresh({ @@ -70,7 +70,7 @@ describe.skip('generate peers', function () { routingTable: table, // @ts-expect-error not a full implementation peerRouting: {}, - lan: false + logPrefix: '' }) }) diff --git a/packages/kad-dht/test/kad-dht.spec.ts b/packages/kad-dht/test/kad-dht.spec.ts index ef6750c91c..2fb03d9a16 100644 --- a/packages/kad-dht/test/kad-dht.spec.ts +++ b/packages/kad-dht/test/kad-dht.spec.ts @@ -2,6 +2,7 @@ /* eslint max-nested-callbacks: ["error", 8] */ import { CodeError } from '@libp2p/interface' +import { peerIdFromBytes } from '@libp2p/peer-id' import { expect } from 'aegir/chai' import delay from 'delay' import all from 'it-all' @@ -15,7 +16,7 @@ import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import * as c from '../src/constants.js' import { EventTypes, type FinalPeerEvent, MessageType, type QueryEvent, type ValueEvent } from '../src/index.js' -import { MESSAGE_TYPE } from '../src/message/index.js' +import { MessageType as PBMessageType } from '../src/message/dht.js' import { peerResponseEvent } from '../src/query/events.js' import { Libp2pRecord } from '../src/record/index.js' import * as kadUtils from '../src/utils.js' @@ -24,7 +25,7 @@ import { createValues } from './utils/create-values.js' import { countDiffPeers } from './utils/index.js' import { sortClosestPeers } from './utils/sort-closest-peers.js' import { TestDHT } from './utils/test-dht.js' -import type { DefaultDualKadDHT } from '../src/dual-kad-dht.js' +import type { KadDHT } from '../src/kad-dht.js' import type { PeerId } from '@libp2p/interface' import type { CID } from 'multiformats/cid' @@ -100,50 +101,48 @@ describe('KadDHT', () => { provideValidity: 60 * 10 } }) - expect(dht.lan.providers).to.have.property('cleanupInterval', 60) - expect(dht.lan.providers).to.have.property('provideValidity', 60 * 10) - expect(dht.wan.providers).to.have.property('cleanupInterval', 60) - expect(dht.wan.providers).to.have.property('provideValidity', 60 * 10) + expect(dht.providers).to.have.property('cleanupInterval', 60) + expect(dht.providers).to.have.property('provideValidity', 60 * 10) }) }) describe('start and stop', () => { - it('simple with defaults', async () => { - const dht = await tdht.spawn(undefined, false) + it('default mode', async () => { + // off by default + const dht = await tdht.spawn({ clientMode: undefined }, false) - sinon.spy(dht.wan.network, 'start') - sinon.spy(dht.wan.network, 'stop') - sinon.spy(dht.lan.network, 'start') - sinon.spy(dht.lan.network, 'stop') + const registrarHandleSpy = sinon.spy(dht.components.registrar, 'handle') await dht.start() - expect(dht.wan.network.start).to.have.property('calledOnce', true) - expect(dht.lan.network.start).to.have.property('calledOnce', true) + // by default we start in client mode + expect(registrarHandleSpy).to.have.property('callCount', 0) + + await dht.setMode('server') + // now we should be in server mode + expect(registrarHandleSpy).to.have.property('callCount', 1) await dht.stop() - expect(dht.wan.network.stop).to.have.property('calledOnce', true) - expect(dht.lan.network.stop).to.have.property('calledOnce', true) }) it('server mode', async () => { - // Currently off by default - const dht = await tdht.spawn(undefined, false) + // turn client mode off explicitly + const dht = await tdht.spawn({ clientMode: false }, false) const registrarHandleSpy = sinon.spy(dht.components.registrar, 'handle') await dht.start() - // lan dht is always in server mode + // should have started in server mode expect(registrarHandleSpy).to.have.property('callCount', 1) await dht.setMode('server') - // now wan dht should be in server mode too + // we were already in server mode expect(registrarHandleSpy).to.have.property('callCount', 2) await dht.stop() }) it('client mode', async () => { - // Currently on by default + // turn client mode on explicitly const dht = await tdht.spawn({ clientMode: true }, false) const registrarHandleSpy = sinon.spy(dht.components.registrar, 'handle') @@ -151,8 +150,8 @@ describe('KadDHT', () => { await dht.start() await dht.stop() - // lan dht is always in server mode, wan is not - expect(registrarHandleSpy).to.have.property('callCount', 1) + // should not have registered handler in client mode + expect(registrarHandleSpy).to.have.property('callCount', 0) }) it('should not fail when already started', async () => { @@ -358,7 +357,7 @@ describe('KadDHT', () => { tdht.spawn() ]) - const dhtASpy = sinon.spy(dhtA.lan.network, 'sendRequest') + const dhtASpy = sinon.spy(dhtA.network, 'sendRequest') // Put before peers connected await drain(dhtA.put(key, valueA)) @@ -428,9 +427,9 @@ describe('KadDHT', () => { const dht = await tdht.spawn() // Simulate returning a peer id to query - sinon.stub(dht.lan.routingTable, 'closestPeers').returns([peerIds[1]]) + sinon.stub(dht.routingTable, 'closestPeers').returns([peerIds[1]]) // Simulate going out to the network and returning the record - sinon.stub(dht.lan.peerRouting, 'getValueOrPeers').callsFake(async function * (peer) { + sinon.stub(dht.peerRouting, 'getValueOrPeers').callsFake(async function * (peer) { yield peerResponseEvent({ messageType: MessageType.GET_VALUE, from: peer, @@ -456,7 +455,7 @@ describe('KadDHT', () => { const ids = dhts.map((d) => d.components.peerId) const idsB58 = ids.map(id => id.toString()) - sinon.spy(dhts[3].lan.network, 'sendMessage') + sinon.spy(dhts[3].network, 'sendMessage') // connect peers await Promise.all([ @@ -469,17 +468,17 @@ describe('KadDHT', () => { await Promise.all(values.map(async (value) => { await drain(dhts[3].provide(value.cid)) })) // Expect an ADD_PROVIDER message to be sent to each peer for each value - const fn = dhts[3].lan.network.sendMessage + const fn = dhts[3].network.sendMessage const valuesBuffs = values.map(v => v.cid.multihash.bytes) // @ts-expect-error fn is a spy const calls = fn.getCalls().map(c => c.args) for (const [peerId, msg] of calls) { expect(idsB58).includes(peerId.toString()) - expect(msg.type).equals(MESSAGE_TYPE.ADD_PROVIDER) + expect(msg.type).equals(PBMessageType.ADD_PROVIDER) expect(valuesBuffs).includes(msg.key) - expect(msg.providerPeers.length).equals(1) - expect(msg.providerPeers[0].id.toString()).equals(idsB58[3]) + expect(msg.providers.length).equals(1) + expect(peerIdFromBytes(msg.providers[0].id).toString()).equals(idsB58[3]) } // Expect each DHT to find the provider of each value @@ -503,31 +502,7 @@ describe('KadDHT', () => { } }) - it('does not provide to wan if in client mode', async function () { - const dhts = await Promise.all([ - tdht.spawn(), - tdht.spawn(), - tdht.spawn(), - tdht.spawn() - ]) - - // connect peers - await Promise.all([ - tdht.connect(dhts[0], dhts[1]), - tdht.connect(dhts[1], dhts[2]), - tdht.connect(dhts[2], dhts[3]) - ]) - - const wanSpy = sinon.spy(dhts[0].wan, 'provide') - const lanSpy = sinon.spy(dhts[0].lan, 'provide') - - await drain(dhts[0].provide(values[0].cid)) - - expect(wanSpy.called).to.be.false() - expect(lanSpy.called).to.be.true() - }) - - it('provides to wan if in server mode', async function () { + it('provides if in server mode', async function () { const dhts = await Promise.all([ tdht.spawn(), tdht.spawn(), @@ -542,15 +517,13 @@ describe('KadDHT', () => { tdht.connect(dhts[2], dhts[3]) ]) - const wanSpy = sinon.spy(dhts[0].wan, 'provide') - const lanSpy = sinon.spy(dhts[0].lan, 'provide') + const sendMessageSpy = sinon.spy(dhts[0].network, 'sendMessage') await dhts[0].setMode('server') await drain(dhts[0].provide(values[0].cid)) - expect(wanSpy.called).to.be.true() - expect(lanSpy.called).to.be.true() + expect(sendMessageSpy.called).to.be.true() }) it('find providers', async function () { @@ -673,7 +646,7 @@ describe('KadDHT', () => { tags: new Map(), metadata: new Map() }) - sinon.stub(dht.lan.providers, 'getProviders').resolves([dht.components.peerId]) + sinon.stub(dht.providers, 'getProviders').resolves([dht.components.peerId]) // Find provider const events = await all(dht.findProviders(val.cid)) @@ -724,7 +697,7 @@ describe('KadDHT', () => { new Array(nDHTs).fill(0).map(async () => tdht.spawn()) ) - const dhtsById = new Map(dhts.map((d) => [d.components.peerId, d])) + const dhtsById = new Map(dhts.map((d) => [d.components.peerId, d])) const ids = [...dhtsById.keys()] // The origin node for the FIND_PEER query @@ -738,7 +711,7 @@ describe('KadDHT', () => { // Make connections between nodes close to each other const sorted = await sortClosestPeers(ids, rtval) - const conns = [] + const conns: PeerId[][] = [] const maxRightIndex = sorted.length - 1 for (let i = 0; i < sorted.length; i++) { // Connect to 5 nodes on either side (10 in total) @@ -768,7 +741,7 @@ describe('KadDHT', () => { // Get the alpha (3) closest peers to the key from the origin's // routing table - const rtablePeers = originNode.lan.routingTable.closestPeers(rtval, c.ALPHA) + const rtablePeers = originNode.routingTable.closestPeers(rtval, c.ALPHA) expect(rtablePeers).to.have.length(c.ALPHA) // The set of peers used to initiate the query (the closest alpha @@ -829,16 +802,6 @@ describe('KadDHT', () => { }) describe('errors', () => { - it('get should fail if only has one peer', async function () { - this.timeout(240 * 1000) - - const dht = await tdht.spawn() - - await delay(100) - - await expect(all(dht.get(uint8ArrayFromString('/v/hello')))).to.eventually.be.rejected().property('code', 'ERR_NO_PEERS_IN_ROUTING_TABLE') - }) - it('get should handle correctly an unexpected error', async function () { this.timeout(240 * 1000) @@ -856,34 +819,10 @@ describe('KadDHT', () => { const errors = await all(filter(dhtA.get(uint8ArrayFromString('/v/hello')), event => event.name === 'QUERY_ERROR')) - expect(errors).to.have.lengthOf(2) + expect(errors).to.have.lengthOf(1) expect(errors).to.have.nested.property('[0].error.code', errCode) - expect(errors).to.have.nested.property('[1].error.code', 'ERR_NOT_FOUND') stub.restore() }) - - it('findPeer should fail if no closest peers available', async function () { - this.timeout(240 * 1000) - - const dhts = await Promise.all([ - tdht.spawn(), - tdht.spawn(), - tdht.spawn(), - tdht.spawn() - ]) - - const ids = dhts.map((d) => d.components.peerId) - await Promise.all([ - tdht.connect(dhts[0], dhts[1]), - tdht.connect(dhts[1], dhts[2]), - tdht.connect(dhts[2], dhts[3]) - ]) - - dhts[0].lan.findPeer = sinon.stub().returns([]) - dhts[0].wan.findPeer = sinon.stub().returns([]) - - await expect(drain(dhts[0].findPeer(ids[3]))).to.eventually.be.rejected().property('code', 'ERR_LOOKUP_FAILED') - }) }) }) diff --git a/packages/kad-dht/test/kad-utils.spec.ts b/packages/kad-dht/test/kad-utils.spec.ts index 4f27ae4a27..137f774002 100644 --- a/packages/kad-dht/test/kad-utils.spec.ts +++ b/packages/kad-dht/test/kad-utils.spec.ts @@ -73,7 +73,7 @@ describe('kad utils', () => { multiaddr('/dns4/localhost/tcp/4001') ] - const peerInfo = utils.removePrivateAddresses({ id, multiaddrs }) + const peerInfo = utils.removePrivateAddressesMapper({ id, multiaddrs }) expect(peerInfo.multiaddrs.map((ma) => ma.toString())) .to.eql(['/dns4/example.com/tcp/4001', '/ip4/1.1.1.1/tcp/4001']) }) @@ -90,7 +90,7 @@ describe('kad utils', () => { multiaddr('/dns4/localhost/tcp/4001') ] - const peerInfo = utils.removePublicAddresses({ id, multiaddrs }) + const peerInfo = utils.removePublicAddressesMapper({ id, multiaddrs }) expect(peerInfo.multiaddrs.map((ma) => ma.toString())) .to.eql(['/ip4/192.168.0.1/tcp/4001', '/dns4/localhost/tcp/4001']) }) diff --git a/packages/kad-dht/test/libp2p-routing.spec.ts b/packages/kad-dht/test/libp2p-routing.spec.ts new file mode 100644 index 0000000000..021e207435 --- /dev/null +++ b/packages/kad-dht/test/libp2p-routing.spec.ts @@ -0,0 +1,801 @@ +import { contentRoutingSymbol, TypedEventEmitter, start, stop, peerRoutingSymbol } from '@libp2p/interface' +import { defaultLogger } from '@libp2p/logger' +import { multiaddr } from '@multiformats/multiaddr' +import { expect } from 'aegir/chai' +import { MemoryDatastore } from 'datastore-core' +import all from 'it-all' +import map from 'it-map' +import { duplexPair } from 'it-pair/duplex' +import { pbStream } from 'it-protobuf-stream' +import { CID } from 'multiformats/cid' +import pDefer from 'p-defer' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { kadDHT, passthroughMapper, type KadDHT } from '../src/index.js' +import { Message, MessageType } from '../src/message/dht.js' +import { convertBuffer } from '../src/utils.js' +import { matchPeerId } from './fixtures/match-peer-id.js' +import { createPeerIds } from './utils/create-peer-id.js' +import { sortClosestPeers } from './utils/sort-closest-peers.js' +import type { ContentRouting, PeerStore, PeerId, TypedEventTarget, ComponentLogger, Connection, Peer, Stream, PeerRouting } from '@libp2p/interface' +import type { AddressManager, ConnectionManager, Registrar } from '@libp2p/interface-internal' +import type { Datastore } from 'interface-datastore' + +interface StubbedKadDHTComponents { + peerId: PeerId + registrar: StubbedInstance + addressManager: StubbedInstance + peerStore: StubbedInstance + connectionManager: StubbedInstance + datastore: Datastore + events: TypedEventTarget + logger: ComponentLogger +} + +const PROTOCOL = '/test/dht/1.0.0' + +function createStreams (peerId: PeerId, components: StubbedKadDHTComponents): { connection: Connection, incomingStream: Stream } { + const duplex = duplexPair() + const outgoingStream = stubInterface() + outgoingStream.source = duplex[0].source + outgoingStream.sink.callsFake(async source => duplex[0].sink(source)) + + const incomingStream = stubInterface() + incomingStream.source = duplex[1].source + incomingStream.sink.callsFake(async source => duplex[1].sink(source)) + + const connection = stubInterface() + connection.newStream.withArgs(PROTOCOL).resolves(outgoingStream) + components.connectionManager.openConnection.withArgs(matchPeerId(peerId)).resolves(connection) + + return { + connection, + incomingStream + } +} + +function createPeer (peerId: PeerId, peer: Partial = {}): Peer { + const minPort = 1000 + const maxPort = 50000 + + return { + id: peerId, + addresses: [{ + isCertified: false, + multiaddr: multiaddr(`/ip4/58.42.62.62/tcp/${Math.random() * (maxPort - minPort) + minPort}`) + }], + tags: new Map(), + metadata: new Map(), + protocols: [ + PROTOCOL + ], + ...peer + } +} + +describe('content routing', () => { + let contentRouting: ContentRouting + let components: StubbedKadDHTComponents + let dht: KadDHT + let peers: PeerId[] + let key: CID + + beforeEach(async () => { + key = CID.parse('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') + + const unsortedPeers = await createPeerIds(5) + + // sort remaining peers by XOR distance to the key, closest -> furthest + peers = await sortClosestPeers(unsortedPeers, await convertBuffer(key.multihash.bytes)) + + components = { + peerId: peers[peers.length - 1], + registrar: stubInterface(), + addressManager: stubInterface(), + peerStore: stubInterface(), + connectionManager: stubInterface(), + datastore: new MemoryDatastore(), + events: new TypedEventEmitter(), + logger: defaultLogger() + } + + dht = kadDHT({ + protocol: PROTOCOL, + peerInfoMapper: passthroughMapper, + clientMode: false, + allowQueryWithZeroPeers: true + })(components) + + await start(dht) + + // @ts-expect-error cannot use symbol to index KadDHT type + contentRouting = dht[contentRoutingSymbol] + }) + + afterEach(async () => { + await stop(dht) + }) + + it('should provide', async () => { + const remotePeer = createPeer(peers[0]) + + components.peerStore.get.withArgs(matchPeerId(remotePeer.id)).resolves(remotePeer) + + const { + connection, + incomingStream + } = createStreams(remotePeer.id, components) + + // a peer has connected + const topology = components.registrar.register.getCall(0).args[1] + topology.onConnect?.(remotePeer.id, connection) + + // begin provide + void contentRouting.provide(key) + + // read FIND_NODE message + const pb = pbStream(incomingStream) + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(key.multihash.bytes) + + // reply with this node + await pb.write({ + type: MessageType.FIND_NODE, + closer: [{ + id: remotePeer.id.toBytes(), + multiaddrs: remotePeer.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + // read ADD_PROVIDER message + const addProviderRequest = await pb.read(Message) + expect(addProviderRequest.type).to.equal(MessageType.ADD_PROVIDER) + }) + + it('should find providers', async () => { + const remotePeer = createPeer(peers[3]) + const providerPeer = createPeer(peers[2]) + + components.peerStore.get.withArgs(matchPeerId(remotePeer.id)).resolves(remotePeer) + + const { + connection, + incomingStream + } = createStreams(remotePeer.id, components) + + // a peer has connected + const topology = components.registrar.register.getCall(0).args[1] + topology.onConnect?.(remotePeer.id, connection) + + void Promise.resolve().then(async () => { + const pb = pbStream(incomingStream) + + // read GET_PROVIDERS message + const getProvidersRequest = await pb.read(Message) + + expect(getProvidersRequest.type).to.equal(MessageType.GET_PROVIDERS) + expect(getProvidersRequest.key).to.equalBytes(key.multihash.bytes) + + // reply with the provider node + await pb.write({ + type: MessageType.GET_PROVIDERS, + providers: [{ + id: providerPeer.id.toBytes(), + multiaddrs: providerPeer.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + }) + + // should have received the provider + await expect(all(map(contentRouting.findProviders(key), prov => ({ + id: prov.id.toString(), + multiaddrs: prov.multiaddrs.map(ma => ma.toString()) + })))).to.eventually.deep.equal([{ + id: providerPeer.id.toString(), + multiaddrs: providerPeer.addresses.map(({ multiaddr }) => multiaddr.toString()) + }]) + }) + + it('should not block on finding providers without multiaddrs', async () => { + const receivedFirstProvider = pDefer() + const remotePeerInteractionsComplete = pDefer() + const providerPeerInteractionsComplete = pDefer() + + const remotePeer = createPeer(peers[3]) + const providerPeerWithoutAddresses = createPeer(peers[2]) + const providerPeer = createPeer(peers[1]) + + components.peerStore.get.withArgs(matchPeerId(remotePeer.id)).resolves(remotePeer) + + const { + connection, + incomingStream + } = createStreams(remotePeer.id, components) + + const { + incomingStream: providerPeerIncomingStream + } = createStreams(providerPeer.id, components) + + // a peer has connected + const topology = components.registrar.register.getCall(0).args[1] + topology.onConnect?.(remotePeer.id, connection) + + // remotePeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(incomingStream) + + // read GET_PROVIDERS message + const getProvidersRequest = await pb.read(Message) + + expect(getProvidersRequest.type).to.equal(MessageType.GET_PROVIDERS) + expect(getProvidersRequest.key).to.equalBytes(key.multihash.bytes) + + // reply with the provider node + await pb.write({ + type: MessageType.GET_PROVIDERS, + providers: [{ + id: providerPeerWithoutAddresses.id.toBytes(), + multiaddrs: [] + }, { + id: providerPeer.id.toBytes(), + multiaddrs: providerPeer.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(providerPeerWithoutAddresses.id.toBytes()) + + // delay sending the response until providerPeer has been received + await receivedFirstProvider.promise + + // return details of providerPeerWithoutAddresses + await pb.write({ + type: MessageType.FIND_NODE, + closer: [{ + id: providerPeerWithoutAddresses.id.toBytes(), + multiaddrs: providerPeerWithoutAddresses.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + remotePeerInteractionsComplete.resolve() + }) + + // providerPeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(providerPeerIncomingStream) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(providerPeerWithoutAddresses.id.toBytes()) + + // don't know providerPeerWithoutAddresses + await pb.write({ + type: MessageType.FIND_NODE + }, Message) + + providerPeerInteractionsComplete.resolve() + }) + + const provs: Array<{ id: string, multiaddrs: string[] }> = [] + + for await (const prov of contentRouting.findProviders(key)) { + provs.push({ + id: prov.id.toString(), + multiaddrs: prov.multiaddrs.map(ma => ma.toString()) + }) + + receivedFirstProvider.resolve() + } + + // should have received the provider + expect(provs).to.deep.equal([{ + id: providerPeer.id.toString(), + multiaddrs: providerPeer.addresses.map(({ multiaddr }) => multiaddr.toString()) + }, { + id: providerPeerWithoutAddresses.id.toString(), + multiaddrs: providerPeerWithoutAddresses.addresses.map(({ multiaddr }) => multiaddr.toString()) + }]) + + await expect(remotePeerInteractionsComplete.promise).to.eventually.be.undefined() + await expect(providerPeerInteractionsComplete.promise).to.eventually.be.undefined() + }) + + it('should ignore providers without multiaddrs', async () => { + const receivedFirstProvider = pDefer() + const remotePeerInteractionsComplete = pDefer() + const providerPeerInteractionsComplete = pDefer() + + const remotePeer = createPeer(peers[3]) + const providerPeerWithoutAddresses = createPeer(peers[2], { + addresses: [] + }) + const providerPeer = createPeer(peers[1]) + + components.peerStore.get.withArgs(matchPeerId(remotePeer.id)).resolves(remotePeer) + + const { + connection, + incomingStream + } = createStreams(remotePeer.id, components) + + const { + incomingStream: providerPeerIncomingStream + } = createStreams(providerPeer.id, components) + + // a peer has connected + const topology = components.registrar.register.getCall(0).args[1] + topology.onConnect?.(remotePeer.id, connection) + + // remotePeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(incomingStream) + + // read GET_PROVIDERS message + const getProvidersRequest = await pb.read(Message) + + expect(getProvidersRequest.type).to.equal(MessageType.GET_PROVIDERS) + expect(getProvidersRequest.key).to.equalBytes(key.multihash.bytes) + + // reply with the provider node + await pb.write({ + type: MessageType.GET_PROVIDERS, + providers: [{ + id: providerPeerWithoutAddresses.id.toBytes(), + multiaddrs: [] + }, { + id: providerPeer.id.toBytes(), + multiaddrs: providerPeer.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(providerPeerWithoutAddresses.id.toBytes()) + + // delay sending the response until providerPeer has been received + await receivedFirstProvider.promise + + // don't know providerPeerWithoutAddresses + await pb.write({ + type: MessageType.FIND_NODE + }, Message) + + remotePeerInteractionsComplete.resolve() + }) + + // providerPeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(providerPeerIncomingStream) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(providerPeerWithoutAddresses.id.toBytes()) + + // don't know providerPeerWithoutAddresses + await pb.write({ + type: MessageType.FIND_NODE + }, Message) + + providerPeerInteractionsComplete.resolve() + }) + + const provs: Array<{ id: string, multiaddrs: string[] }> = [] + + for await (const prov of contentRouting.findProviders(key)) { + provs.push({ + id: prov.id.toString(), + multiaddrs: prov.multiaddrs.map(ma => ma.toString()) + }) + + receivedFirstProvider.resolve() + } + + // should have received the provider + expect(provs).to.deep.equal([{ + id: providerPeer.id.toString(), + multiaddrs: providerPeer.addresses.map(({ multiaddr }) => multiaddr.toString()) + }]) + + await expect(remotePeerInteractionsComplete.promise).to.eventually.be.undefined() + await expect(providerPeerInteractionsComplete.promise).to.eventually.be.undefined() + }) +}) + +describe('peer routing', () => { + let peerRouting: PeerRouting + let components: StubbedKadDHTComponents + let dht: KadDHT + let peers: PeerId[] + let key: CID + + beforeEach(async () => { + key = CID.parse('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') + + const unsortedPeers = await createPeerIds(5) + + // sort remaining peers by XOR distance to the key, closest -> furthest + peers = await sortClosestPeers(unsortedPeers, await convertBuffer(key.multihash.bytes)) + + components = { + peerId: peers[peers.length - 1], + registrar: stubInterface(), + addressManager: stubInterface(), + peerStore: stubInterface(), + connectionManager: stubInterface(), + datastore: new MemoryDatastore(), + events: new TypedEventEmitter(), + logger: defaultLogger() + } + + dht = kadDHT({ + protocol: PROTOCOL, + peerInfoMapper: passthroughMapper, + clientMode: false, + allowQueryWithZeroPeers: true + })(components) + + await start(dht) + + // @ts-expect-error cannot use symbol to index KadDHT type + peerRouting = dht[peerRoutingSymbol] + }) + + afterEach(async () => { + await stop(dht) + }) + + it('should find peer', async () => { + const remotePeer = createPeer(peers[1]) + const targetPeer = createPeer(peers[0]) + + components.peerStore.get.withArgs(matchPeerId(remotePeer.id)).resolves(remotePeer) + + const { + connection, + incomingStream + } = createStreams(remotePeer.id, components) + + // a peer has connected + const topology = components.registrar.register.getCall(0).args[1] + topology.onConnect?.(remotePeer.id, connection) + + // begin find + const p = peerRouting.findPeer(peers[0]) + + // read FIND_NODE message + const pb = pbStream(incomingStream) + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(peers[0].toBytes()) + + // reply with this node + await pb.write({ + type: MessageType.FIND_NODE, + closer: [{ + id: targetPeer.id.toBytes(), + multiaddrs: targetPeer.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + const peerInfo = await p + + expect({ + id: peerInfo.id.toString(), + multiaddrs: peerInfo.multiaddrs.map(ma => ma.toString()) + }).to.deep.equal({ + id: targetPeer.id.toString(), + multiaddrs: targetPeer.addresses.map(({ multiaddr }) => multiaddr.toString()) + }) + }) + + it('should find closest peers', async () => { + const remotePeer = createPeer(peers[3]) + const closestPeer = createPeer(peers[2]) + + const remotePeerInteractionsComplete = pDefer() + const closestPeerInteractionsComplete = pDefer() + + components.peerStore.get.withArgs(matchPeerId(remotePeer.id)).resolves(remotePeer) + components.peerStore.get.withArgs(matchPeerId(closestPeer.id)).resolves(closestPeer) + + const { + connection, + incomingStream + } = createStreams(remotePeer.id, components) + + const { + incomingStream: closestPeerIncomingStream + } = createStreams(closestPeer.id, components) + + // a peer has connected + const topology = components.registrar.register.getCall(0).args[1] + topology.onConnect?.(remotePeer.id, connection) + + // remotePeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(incomingStream) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(key.multihash.bytes) + + // reply with the closest node + await pb.write({ + type: MessageType.FIND_NODE, + closer: [{ + id: closestPeer.id.toBytes(), + multiaddrs: closestPeer.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + remotePeerInteractionsComplete.resolve() + }) + + // closestPeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(closestPeerIncomingStream) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(key.multihash.bytes) + + // we are the closest so no closer peers + await pb.write({ + type: MessageType.FIND_NODE + }, Message) + + closestPeerInteractionsComplete.resolve() + }) + + // should have received the closest peer + await expect(all(map(peerRouting.getClosestPeers(key.multihash.bytes), prov => ({ + id: prov.id.toString(), + multiaddrs: prov.multiaddrs.map(ma => ma.toString()) + })))).to.eventually.deep.equal([{ + id: closestPeer.id.toString(), + multiaddrs: closestPeer.addresses.map(({ multiaddr }) => multiaddr.toString()) + }]) + + await expect(remotePeerInteractionsComplete.promise).to.eventually.be.undefined() + await expect(closestPeerInteractionsComplete.promise).to.eventually.be.undefined() + }) + + it('should not block on finding closest peers without multiaddrs', async () => { + const receivedFirstClosest = pDefer() + const remotePeerInteractionsComplete = pDefer() + const closestPeerInteractionsComplete = pDefer() + + const remotePeer = createPeer(peers[3]) + const closestPeerWithoutAddresses = createPeer(peers[2]) + const closestPeer = createPeer(peers[1]) + + components.peerStore.get.withArgs(matchPeerId(remotePeer.id)).resolves(remotePeer) + components.peerStore.get.withArgs(matchPeerId(closestPeer.id)).resolves(closestPeer) + components.peerStore.get.withArgs(matchPeerId(closestPeerWithoutAddresses.id)).resolves({ + ...closestPeerWithoutAddresses, + addresses: [] + }) + + const { + connection, + incomingStream + } = createStreams(remotePeer.id, components) + + const { + incomingStream: closestPeerIncomingStream + } = createStreams(closestPeer.id, components) + + // a peer has connected + const topology = components.registrar.register.getCall(0).args[1] + topology.onConnect?.(remotePeer.id, connection) + + // remotePeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(incomingStream) + + // read FIND_NODE message + const getProvidersRequest = await pb.read(Message) + + expect(getProvidersRequest.type).to.equal(MessageType.FIND_NODE) + expect(getProvidersRequest.key).to.equalBytes(key.multihash.bytes) + + // reply with the closer nodes + await pb.write({ + type: MessageType.FIND_NODE, + closer: [{ + id: closestPeerWithoutAddresses.id.toBytes(), + multiaddrs: [] + }, { + id: closestPeer.id.toBytes(), + multiaddrs: closestPeer.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(closestPeerWithoutAddresses.id.toBytes()) + + // delay sending the response until closestPeer has been received + await receivedFirstClosest.promise + + // return details of closestPeerWithoutAddresses + await pb.write({ + type: MessageType.FIND_NODE, + closer: [{ + id: closestPeerWithoutAddresses.id.toBytes(), + multiaddrs: closestPeerWithoutAddresses.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + remotePeerInteractionsComplete.resolve() + }) + + // closestPeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(closestPeerIncomingStream) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(key.multihash.bytes) + + await pb.write({ + type: MessageType.FIND_NODE, + closer: [] + }, Message) + + const secondFindNodeRequest = await pb.read(Message) + expect(secondFindNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(secondFindNodeRequest.key).to.equalBytes(closestPeerWithoutAddresses.id.toBytes()) + + // don't know closestPeerWithoutAddresses + await pb.write({ + type: MessageType.FIND_NODE + }, Message) + + closestPeerInteractionsComplete.resolve() + }) + + const closest: Array<{ id: string, multiaddrs: string[] }> = [] + + for await (const closer of peerRouting.getClosestPeers(key.multihash.bytes)) { + closest.push({ + id: closer.id.toString(), + multiaddrs: closer.multiaddrs.map(ma => ma.toString()) + }) + + receivedFirstClosest.resolve() + } + + // should have received the closest peers + expect(closest).to.deep.equal([{ + id: closestPeer.id.toString(), + multiaddrs: closestPeer.addresses.map(({ multiaddr }) => multiaddr.toString()) + }, { + id: closestPeerWithoutAddresses.id.toString(), + multiaddrs: closestPeerWithoutAddresses.addresses.map(({ multiaddr }) => multiaddr.toString()) + }]) + + await expect(remotePeerInteractionsComplete.promise).to.eventually.be.undefined() + await expect(closestPeerInteractionsComplete.promise).to.eventually.be.undefined() + }) + + it('should ignore closest peers without multiaddrs', async () => { + const receivedFirstClosest = pDefer() + const remotePeerInteractionsComplete = pDefer() + const closestPeerInteractionsComplete = pDefer() + + const remotePeer = createPeer(peers[3]) + const closestPeerWithoutAddresses = createPeer(peers[2]) + const closestPeer = createPeer(peers[1]) + + components.peerStore.get.withArgs(matchPeerId(remotePeer.id)).resolves(remotePeer) + components.peerStore.get.withArgs(matchPeerId(closestPeer.id)).resolves(closestPeer) + components.peerStore.get.withArgs(matchPeerId(closestPeerWithoutAddresses.id)).resolves({ + ...closestPeerWithoutAddresses, + addresses: [] + }) + + const { + connection, + incomingStream + } = createStreams(remotePeer.id, components) + + const { + incomingStream: closestPeerIncomingStream + } = createStreams(closestPeer.id, components) + + // a peer has connected + const topology = components.registrar.register.getCall(0).args[1] + topology.onConnect?.(remotePeer.id, connection) + + // remotePeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(incomingStream) + + // read FIND_NODE message + const getProvidersRequest = await pb.read(Message) + + expect(getProvidersRequest.type).to.equal(MessageType.FIND_NODE) + expect(getProvidersRequest.key).to.equalBytes(key.multihash.bytes) + + // reply with the closer nodes + await pb.write({ + type: MessageType.FIND_NODE, + closer: [{ + id: closestPeerWithoutAddresses.id.toBytes(), + multiaddrs: [] + }, { + id: closestPeer.id.toBytes(), + multiaddrs: closestPeer.addresses.map(({ multiaddr }) => multiaddr.bytes) + }] + }, Message) + + const secondFindNodeRequest = await pb.read(Message) + expect(secondFindNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(secondFindNodeRequest.key).to.equalBytes(closestPeerWithoutAddresses.id.toBytes()) + + // don't know closestPeerWithoutAddresses + await pb.write({ + type: MessageType.FIND_NODE + }, Message) + + remotePeerInteractionsComplete.resolve() + }) + + // closestPeer stream + void Promise.resolve().then(async () => { + const pb = pbStream(closestPeerIncomingStream) + + // read FIND_NODE message + const findNodeRequest = await pb.read(Message) + expect(findNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(findNodeRequest.key).to.equalBytes(key.multihash.bytes) + + await pb.write({ + type: MessageType.FIND_NODE, + closer: [] + }, Message) + + const secondFindNodeRequest = await pb.read(Message) + expect(secondFindNodeRequest.type).to.equal(MessageType.FIND_NODE) + expect(secondFindNodeRequest.key).to.equalBytes(closestPeerWithoutAddresses.id.toBytes()) + + // don't know closestPeerWithoutAddresses + await pb.write({ + type: MessageType.FIND_NODE + }, Message) + + closestPeerInteractionsComplete.resolve() + }) + + const closest: Array<{ id: string, multiaddrs: string[] }> = [] + + for await (const closer of peerRouting.getClosestPeers(key.multihash.bytes)) { + closest.push({ + id: closer.id.toString(), + multiaddrs: closer.multiaddrs.map(ma => ma.toString()) + }) + + receivedFirstClosest.resolve() + } + + // should have received the closest peers + expect(closest).to.deep.equal([{ + id: closestPeer.id.toString(), + multiaddrs: closestPeer.addresses.map(({ multiaddr }) => multiaddr.toString()) + }]) + + await expect(remotePeerInteractionsComplete.promise).to.eventually.be.undefined() + await expect(closestPeerInteractionsComplete.promise).to.eventually.be.undefined() + }) +}) diff --git a/packages/kad-dht/test/message.node.ts b/packages/kad-dht/test/message.node.ts index 20bb8729ac..6a21c93386 100644 --- a/packages/kad-dht/test/message.node.ts +++ b/packages/kad-dht/test/message.node.ts @@ -3,9 +3,11 @@ import fs from 'fs' import path from 'path' import { isPeerId } from '@libp2p/interface' +import { peerIdFromBytes } from '@libp2p/peer-id' import { expect } from 'aegir/chai' import range from 'lodash.range' -import { Message } from '../src/message/index.js' +import { Message } from '../src/message/dht.js' +import { Libp2pRecord } from '../src/record/index.js' describe('Message', () => { it('go-interop', () => { @@ -14,16 +16,18 @@ describe('Message', () => { path.join(process.cwd(), 'test', 'fixtures', `msg-${i}`) ) - const msg = Message.deserialize(raw) + const msg = Message.decode(raw) expect(msg.clusterLevel).to.gte(0) if (msg.record != null) { - expect(msg.record.key).to.be.a('Uint8Array') + const record = Libp2pRecord.deserialize(msg.record) + + expect(record.key).to.be.a('Uint8Array') } - if (msg.providerPeers.length > 0) { - msg.providerPeers.forEach((p) => { - expect(isPeerId(p.id)).to.be.true() + if (msg.providers.length > 0) { + msg.providers.forEach((p) => { + expect(isPeerId(peerIdFromBytes(p.id))).to.be.true() }) } }) diff --git a/packages/kad-dht/test/message.spec.ts b/packages/kad-dht/test/message.spec.ts index eee15095c2..006f3b7c44 100644 --- a/packages/kad-dht/test/message.spec.ts +++ b/packages/kad-dht/test/message.spec.ts @@ -5,20 +5,11 @@ import { multiaddr } from '@multiformats/multiaddr' import { expect } from 'aegir/chai' import random from 'lodash.random' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Message, MESSAGE_TYPE } from '../src/message/index.js' +import { Message, MessageType } from '../src/message/dht.js' +import { toPbPeerInfo } from '../src/message/utils.js' import { Libp2pRecord } from '../src/record/index.js' describe('Message', () => { - it('create', () => { - const k = uint8ArrayFromString('hello') - const msg = new Message(MESSAGE_TYPE.PING, k, 5) - - expect(msg).to.have.property('type', 'PING') - expect(msg).to.have.property('key').eql(uint8ArrayFromString('hello')) - expect(msg).to.have.property('clusterLevelRaw', 5) - expect(msg).to.have.property('clusterLevel', 4) - }) - it('serialize & deserialize', async function () { this.timeout(10 * 1000) @@ -30,28 +21,29 @@ describe('Message', () => { multiaddrs: [ multiaddr(`/ip4/198.176.1.${random(198)}/tcp/1234`), multiaddr(`/ip4/100.176.1.${random(198)}`) - ], - protocols: [] + ] })) - const provider = peers.slice(0, 5).map((p) => ({ + const providers = peers.slice(0, 5).map((p) => ({ id: p, multiaddrs: [ multiaddr(`/ip4/98.176.1.${random(198)}/tcp/1234`), multiaddr(`/ip4/10.176.1.${random(198)}`) - ], - protocols: [] + ] })) - const msg = new Message(MESSAGE_TYPE.GET_VALUE, uint8ArrayFromString('hello'), 5) const record = new Libp2pRecord(uint8ArrayFromString('hello'), uint8ArrayFromString('world'), new Date()) - msg.closerPeers = closer - msg.providerPeers = provider - msg.record = record + const msg: Partial = { + type: MessageType.GET_VALUE, + key: uint8ArrayFromString('hello'), + closer: closer.map(peer => toPbPeerInfo(peer)), + providers: providers.map(peer => toPbPeerInfo(peer)), + record: record.serialize() + } - const enc = msg.serialize() - const dec = Message.deserialize(enc) + const enc = Message.encode(msg) + const dec = Message.decode(enc) expect(dec.type).to.be.eql(msg.type) expect(dec.key).to.be.eql(msg.key) @@ -61,26 +53,18 @@ describe('Message', () => { throw new Error('No record found') } - expect(dec.record.serialize()).to.be.eql(record.serialize()) - expect(dec.record.key).to.eql(uint8ArrayFromString('hello')) + expect(dec.record).to.equalBytes(record.serialize()) - expect(dec.closerPeers).to.have.length(5) - dec.closerPeers.forEach((peer, i) => { - expect(peer.id.equals(msg.closerPeers[i].id)).to.eql(true) - expect(peer.multiaddrs).to.eql(msg.closerPeers[i].multiaddrs) + expect(dec.closer).to.have.length(5) + dec.closer.forEach((peer, i) => { + expect(peer.id).equalBytes(msg.closer?.[i].id) + expect(peer.multiaddrs).to.deep.equal(msg.closer?.[i].multiaddrs) }) - expect(dec.providerPeers).to.have.length(5) - dec.providerPeers.forEach((peer, i) => { - expect(peer.id.equals(msg.providerPeers[i].id)).to.equal(true) - expect(peer.multiaddrs).to.eql(msg.providerPeers[i].multiaddrs) + expect(dec.providers).to.have.length(5) + dec.providers.forEach((peer, i) => { + expect(peer.id).equalBytes(msg.providers?.[i].id) + expect(peer.multiaddrs).to.deep.equal(msg.providers?.[i].multiaddrs) }) }) - - it('clusterlevel', () => { - const msg = new Message(MESSAGE_TYPE.PING, uint8ArrayFromString('hello'), 0) - - msg.clusterLevel = 10 - expect(msg.clusterLevel).to.eql(9) - }) }) diff --git a/packages/kad-dht/test/multiple-nodes.spec.ts b/packages/kad-dht/test/multiple-nodes.spec.ts index 24ecde0dd7..4365d747c7 100644 --- a/packages/kad-dht/test/multiple-nodes.spec.ts +++ b/packages/kad-dht/test/multiple-nodes.spec.ts @@ -5,13 +5,13 @@ import drain from 'it-drain' import last from 'it-last' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { TestDHT } from './utils/test-dht.js' -import type { DefaultDualKadDHT } from '../src/dual-kad-dht.js' +import type { KadDHT } from '../src/kad-dht.js' describe('multiple nodes', function () { this.timeout(60 * 1000) const n = 8 let tdht: TestDHT - let dhts: DefaultDualKadDHT[] + let dhts: KadDHT[] // spawn nodes beforeEach(async function () { diff --git a/packages/kad-dht/test/network.spec.ts b/packages/kad-dht/test/network.spec.ts index 395ab0c138..ebca54ae03 100644 --- a/packages/kad-dht/test/network.spec.ts +++ b/packages/kad-dht/test/network.spec.ts @@ -6,16 +6,16 @@ import all from 'it-all' import * as lp from 'it-length-prefixed' import pDefer from 'p-defer' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Message, MESSAGE_TYPE } from '../src/message/index.js' +import { Message, MessageType } from '../src/message/dht.js' import { TestDHT } from './utils/test-dht.js' -import type { DefaultDualKadDHT } from '../src/dual-kad-dht.js' +import type { KadDHT } from '../src/kad-dht.js' import type { Connection, PeerId } from '@libp2p/interface' import type { Multiaddr } from '@multiformats/multiaddr' import type { Sink, Source } from 'it-stream-types' import type { Uint8ArrayList } from 'uint8arraylist' describe('Network', () => { - let dht: DefaultDualKadDHT + let dht: KadDHT let tdht: TestDHT before(async function () { @@ -30,13 +30,16 @@ describe('Network', () => { describe('sendRequest', () => { it('send and response echo', async () => { - const msg = new Message(MESSAGE_TYPE.PING, uint8ArrayFromString('hello'), 0) + const msg: Partial = { + type: MessageType.PING, + key: uint8ArrayFromString('hello') + } - const events = await all(dht.lan.network.sendRequest(dht.components.peerId, msg)) + const events = await all(dht.network.sendRequest(dht.components.peerId, msg)) const response = events .filter(event => event.name === 'PEER_RESPONSE') .pop() - expect(response).to.have.property('messageType', MESSAGE_TYPE.PING) + expect(response).to.have.property('messageType', MessageType.PING) }) it('send and response different messages', async () => { @@ -48,7 +51,10 @@ describe('Network', () => { } } - const msg = new Message(MESSAGE_TYPE.PING, uint8ArrayFromString('hello'), 0) + const msg: Partial = { + type: MessageType.PING, + key: uint8ArrayFromString('hello') + } // mock it dht.components.connectionManager.openConnection = async (peer: PeerId | Multiaddr | Multiaddr[]) => { @@ -56,15 +62,18 @@ describe('Network', () => { const connection: Connection = { newStream: async (protocols: string | string[]) => { const protocol = Array.isArray(protocols) ? protocols[0] : protocols - const msg = new Message(MESSAGE_TYPE.FIND_NODE, uint8ArrayFromString('world'), 0) + const msg: Partial = { + type: MessageType.FIND_NODE, + key: uint8ArrayFromString('world') + } const source = (async function * () { - yield lp.encode.single(msg.serialize()) + yield lp.encode.single(Message.encode(msg)) })() const sink: Sink, Promise> = async source => { for await (const buf of lp.decode(source)) { - expect(Message.deserialize(buf).type).to.eql(MESSAGE_TYPE.PING) + expect(Message.decode(buf).type).to.eql(MessageType.PING) finish() } } @@ -81,12 +90,12 @@ describe('Network', () => { return connection } - const events = await all(dht.lan.network.sendRequest(dht.components.peerId, msg)) + const events = await all(dht.network.sendRequest(dht.components.peerId, msg)) const response = events .filter(event => event.name === 'PEER_RESPONSE') .pop() - expect(response).to.have.property('messageType', MESSAGE_TYPE.FIND_NODE) + expect(response).to.have.property('messageType', MessageType.FIND_NODE) finish() return defer.promise diff --git a/packages/kad-dht/test/query-self.spec.ts b/packages/kad-dht/test/query-self.spec.ts index b7b7d72f12..bb91d4d2bf 100644 --- a/packages/kad-dht/test/query-self.spec.ts +++ b/packages/kad-dht/test/query-self.spec.ts @@ -35,7 +35,8 @@ describe('Query Self', () => { lan: false, peerRouting, routingTable, - initialQuerySelfHasRun + initialQuerySelfHasRun, + logPrefix: '' } querySelf = new QuerySelf(components, init) diff --git a/packages/kad-dht/test/query.spec.ts b/packages/kad-dht/test/query.spec.ts index 2715119ee9..a3c1b92027 100644 --- a/packages/kad-dht/test/query.spec.ts +++ b/packages/kad-dht/test/query.spec.ts @@ -9,7 +9,7 @@ import pDefer from 'p-defer' import { type StubbedInstance, stubInterface } from 'sinon-ts' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { EventTypes, type QueryEvent } from '../src/index.js' -import { MESSAGE_TYPE } from '../src/message/index.js' +import { MessageType } from '../src/message/dht.js' import { peerResponseEvent, valueEvent, @@ -45,7 +45,8 @@ describe('QueryManager', () => { const defaultInit = (): QueryManagerInit => { const init: QueryManagerInit = { initialQuerySelfHasRun: pDefer(), - routingTable + routingTable, + logPrefix: '' } init.initialQuerySelfHasRun.resolve() @@ -70,7 +71,7 @@ describe('QueryManager', () => { } else { event = peerResponseEvent({ from, - messageType: MESSAGE_TYPE.GET_VALUE, + messageType: MessageType.GET_VALUE, closer: (config.closerPeers ?? []).map((id) => ({ id: peers[id], multiaddrs: [], @@ -196,7 +197,7 @@ describe('QueryManager', () => { }) await manager.start() - const peersQueried = [] + const peersQueried: PeerId[] = [] const queryFunc: QueryFunc = async function * ({ peer, signal }) { // eslint-disable-line require-await expect(signal).to.be.an.instanceOf(AbortSignal) @@ -206,7 +207,7 @@ describe('QueryManager', () => { // query more peers yield peerResponseEvent({ from: peer, - messageType: MESSAGE_TYPE.GET_VALUE, + messageType: MessageType.GET_VALUE, closer: peers.slice(0, 5).map(id => ({ id, multiaddrs: [], protocols: [] })) }) } else if (peersQueried.length === 6) { @@ -219,7 +220,7 @@ describe('QueryManager', () => { // a peer that cannot help in our query yield peerResponseEvent({ from: peer, - messageType: MESSAGE_TYPE.GET_VALUE + messageType: MessageType.GET_VALUE }) } } @@ -249,7 +250,7 @@ describe('QueryManager', () => { }) await manager.start() - const peersQueried = [] + const peersQueried: PeerId[] = [] const queryFunc: QueryFunc = async function * ({ peer }) { // eslint-disable-line require-await peersQueried.push(peer) @@ -258,14 +259,14 @@ describe('QueryManager', () => { // query more peers yield peerResponseEvent({ from: peer, - messageType: MESSAGE_TYPE.GET_VALUE, + messageType: MessageType.GET_VALUE, closer: peers.slice(0, 5).map(id => ({ id, multiaddrs: [], protocols: [] })) }) } else { // a peer that cannot help in our query yield peerResponseEvent({ from: peer, - messageType: MESSAGE_TYPE.GET_VALUE + messageType: MessageType.GET_VALUE }) } } @@ -396,7 +397,7 @@ describe('QueryManager', () => { error: new Error('Urk!') }) } else { - yield peerResponseEvent({ from: peer, messageType: MESSAGE_TYPE.GET_VALUE }) + yield peerResponseEvent({ from: peer, messageType: MessageType.GET_VALUE }) } } @@ -544,7 +545,7 @@ describe('QueryManager', () => { const queryFunc: QueryFunc = async function * ({ peer }) { // eslint-disable-line require-await yield peerResponseEvent({ from: peer, - messageType: MESSAGE_TYPE.GET_VALUE, + messageType: MessageType.GET_VALUE, closer: [{ id: peers[2], multiaddrs: [] @@ -739,7 +740,8 @@ describe('QueryManager', () => { logger: defaultLogger() }, { initialQuerySelfHasRun: pDefer(), - routingTable + routingTable, + logPrefix: '' }) await manager.start() @@ -774,7 +776,8 @@ describe('QueryManager', () => { }, { initialQuerySelfHasRun, alpha: 2, - routingTable + routingTable, + logPrefix: '' }) await manager.start() diff --git a/packages/kad-dht/test/routing-table.spec.ts b/packages/kad-dht/test/routing-table.spec.ts index 3babf30480..96738f8fd8 100644 --- a/packages/kad-dht/test/routing-table.spec.ts +++ b/packages/kad-dht/test/routing-table.spec.ts @@ -10,18 +10,21 @@ import { PersistentPeerStore } from '@libp2p/peer-store' import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' import all from 'it-all' +import drain from 'it-drain' import { pipe } from 'it-pipe' import random from 'lodash.random' import { pEvent } from 'p-event' import pWaitFor from 'p-wait-for' import sinon from 'sinon' import { stubInterface } from 'sinon-ts' -import { PROTOCOL_DHT } from '../src/constants.js' +import { Uint8ArrayList } from 'uint8arraylist' +import { PROTOCOL } from '../src/constants.js' +import { Message, MessageType } from '../src/message/dht.js' import { KAD_CLOSE_TAG_NAME, KAD_CLOSE_TAG_VALUE, KBUCKET_SIZE, RoutingTable, type RoutingTableComponents } from '../src/routing-table/index.js' import * as kadUtils from '../src/utils.js' import { createPeerId, createPeerIds } from './utils/create-peer-id.js' import { sortClosestPeers } from './utils/sort-closest-peers.js' -import type { Libp2pEvents, PeerId, PeerStore } from '@libp2p/interface' +import type { Libp2pEvents, PeerId, PeerStore, Stream } from '@libp2p/interface' import type { ConnectionManager, Registrar } from '@libp2p/interface-internal' describe('Routing Table', () => { @@ -51,8 +54,8 @@ describe('Routing Table', () => { }) table = new RoutingTable(components, { - lan: false, - protocol: PROTOCOL_DHT + logPrefix: '', + protocol: PROTOCOL }) await table.start() }) @@ -138,17 +141,6 @@ describe('Routing Table', () => { }) it('favours old peers that respond to pings', async () => { - let fn: (() => Promise) | undefined - - // execute queued functions immediately - // @ts-expect-error incomplete implementation - table.pingQueue = { - add: async (f: () => Promise) => { - fn = f - }, - clear: () => {} - } - const peerIds = [ peerIdFromString('QmYobx1VAHP7Mi88LcDvLeQoWcc1Aa2rynYHpdEPBqHZi5'), peerIdFromString('QmYobx1VAHP7Mi88LcDvLeQoWcc1Aa2rynYHpdEPBqHZi6') @@ -163,8 +155,6 @@ describe('Routing Table', () => { peer: peerIds[1] } - table._onPing(new CustomEvent('ping', { detail: { oldContacts: [oldPeer], newContact: newPeer } })) - if (table.kb == null) { throw new Error('kbucket not defined') } @@ -172,25 +162,31 @@ describe('Routing Table', () => { // add the old peer table.kb.add(oldPeer) + const stream = stubInterface({ + source: (async function * () { + yield new Uint8ArrayList(Uint8Array.from([2]), Message.encode({ + type: MessageType.PING + })) + })(), + sink: async function (source) { + await drain(source) + } + }) + // simulate connection succeeding - const newStreamStub = sinon.stub().withArgs(PROTOCOL_DHT).resolves({ close: sinon.stub() }) + const newStreamStub = sinon.stub().withArgs(PROTOCOL).resolves(stream) const openConnectionStub = sinon.stub().withArgs(oldPeer.peer).resolves({ newStream: newStreamStub }) components.connectionManager.openConnection = openConnectionStub - if (fn == null) { - throw new Error('nothing added to queue') - } - - // perform the ping - await fn() + await table._onPing(new CustomEvent('ping', { detail: { oldContacts: [oldPeer], newContact: newPeer } })) expect(openConnectionStub.calledOnce).to.be.true() expect(openConnectionStub.calledWith(oldPeer.peer)).to.be.true() expect(newStreamStub.callCount).to.equal(1) - expect(newStreamStub.calledWith(PROTOCOL_DHT)).to.be.true() + expect(newStreamStub.calledWith(PROTOCOL)).to.be.true() // did not add the new peer expect(table.kb.get(newPeer.id)).to.be.undefined() @@ -200,17 +196,6 @@ describe('Routing Table', () => { }) it('evicts oldest peer that does not respond to ping', async () => { - let fn: (() => Promise) | undefined - - // execute queued functions immediately - // @ts-expect-error incomplete implementation - table.pingQueue = { - add: async (f: () => Promise) => { - fn = f - }, - clear: () => {} - } - const peerIds = [ peerIdFromString('QmYobx1VAHP7Mi88LcDvLeQoWcc1Aa2rynYHpdEPBqHZi5'), peerIdFromString('QmYobx1VAHP7Mi88LcDvLeQoWcc1Aa2rynYHpdEPBqHZi6') @@ -225,7 +210,9 @@ describe('Routing Table', () => { peer: peerIds[1] } - table._onPing(new CustomEvent('ping', { detail: { oldContacts: [oldPeer], newContact: newPeer } })) + // libp2p fails to dial the old peer + const openConnectionStub = sinon.stub().withArgs(oldPeer.peer).rejects(new Error('Could not dial peer')) + components.connectionManager.openConnection = openConnectionStub if (table.kb == null) { throw new Error('kbucket not defined') @@ -234,16 +221,8 @@ describe('Routing Table', () => { // add the old peer table.kb.add(oldPeer) - // libp2p fails to dial the old peer - const openConnectionStub = sinon.stub().withArgs(oldPeer.peer).rejects(new Error('Could not dial peer')) - components.connectionManager.openConnection = openConnectionStub - - if (fn == null) { - throw new Error('nothing added to queue') - } - - // perform the ping - await fn() + await table._onPing(new CustomEvent('ping', { detail: { oldContacts: [oldPeer], newContact: newPeer } })) + await table.pingQueue.onIdle() expect(openConnectionStub.callCount).to.equal(1) expect(openConnectionStub.calledWith(oldPeer.peer)).to.be.true() diff --git a/packages/kad-dht/test/rpc/handlers/add-provider.spec.ts b/packages/kad-dht/test/rpc/handlers/add-provider.spec.ts index 64973c2bd0..72adb3aa72 100644 --- a/packages/kad-dht/test/rpc/handlers/add-provider.spec.ts +++ b/packages/kad-dht/test/rpc/handlers/add-provider.spec.ts @@ -5,7 +5,7 @@ import { multiaddr } from '@multiformats/multiaddr' import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Message, MESSAGE_TYPE } from '../../../src/message/index.js' +import { type Message, MessageType } from '../../../src/message/dht.js' import { Providers } from '../../../src/providers.js' import { AddProviderHandler } from '../../../src/rpc/handlers/add-provider.js' import { createPeerIds } from '../../utils/create-peer-id.js' @@ -38,16 +38,27 @@ describe('rpc - handlers - AddProvider', () => { handler = new AddProviderHandler({ logger: defaultLogger() }, { - providers + providers, + logPrefix: '' }) }) describe('invalid messages', () => { - const tests = [{ - message: new Message(MESSAGE_TYPE.ADD_PROVIDER, new Uint8Array(0), 0), + const tests: Array<{ message: Message, error: string }> = [{ + message: { + type: MessageType.ADD_PROVIDER, + key: new Uint8Array(0), + closer: [], + providers: [] + }, error: 'ERR_MISSING_KEY' }, { - message: new Message(MESSAGE_TYPE.ADD_PROVIDER, uint8ArrayFromString('hello world'), 0), + message: { + type: MessageType.ADD_PROVIDER, + key: uint8ArrayFromString('hello world'), + closer: [], + providers: [] + }, error: 'ERR_INVALID_CID' }] @@ -67,17 +78,22 @@ describe('rpc - handlers - AddProvider', () => { it('ignore providers that do not match the sender', async () => { const cid = values[0].cid - const msg = new Message(MESSAGE_TYPE.ADD_PROVIDER, cid.bytes, 0) + const msg: Message = { + type: MessageType.ADD_PROVIDER, + key: cid.bytes, + closer: [], + providers: [] + } const ma1 = multiaddr('/ip4/127.0.0.1/tcp/1234') const ma2 = multiaddr('/ip4/127.0.0.1/tcp/2345') - msg.providerPeers = [{ - id: peerIds[0], - multiaddrs: [ma1] + msg.providers = [{ + id: peerIds[0].toBytes(), + multiaddrs: [ma1.bytes] }, { - id: peerIds[1], - multiaddrs: [ma2] + id: peerIds[1].toBytes(), + multiaddrs: [ma2.bytes] }] await handler.handle(peerIds[0], msg) diff --git a/packages/kad-dht/test/rpc/handlers/find-node.spec.ts b/packages/kad-dht/test/rpc/handlers/find-node.spec.ts index 5ea85b0763..8ca6535470 100644 --- a/packages/kad-dht/test/rpc/handlers/find-node.spec.ts +++ b/packages/kad-dht/test/rpc/handlers/find-node.spec.ts @@ -1,20 +1,22 @@ /* eslint-env mocha */ import { defaultLogger } from '@libp2p/logger' +import { peerIdFromBytes } from '@libp2p/peer-id' import { multiaddr } from '@multiformats/multiaddr' import { expect } from 'aegir/chai' import Sinon, { type SinonStubbedInstance } from 'sinon' import { stubInterface } from 'sinon-ts' -import { Message, MESSAGE_TYPE } from '../../../src/message/index.js' +import { type Message, MessageType } from '../../../src/message/dht.js' import { PeerRouting } from '../../../src/peer-routing/index.js' import { FindNodeHandler } from '../../../src/rpc/handlers/find-node.js' +import { passthroughMapper, removePrivateAddressesMapper, removePublicAddressesMapper } from '../../../src/utils.js' import { createPeerId } from '../../utils/create-peer-id.js' import type { DHTMessageHandler } from '../../../src/rpc/index.js' import type { PeerId } from '@libp2p/interface' import type { AddressManager } from '@libp2p/interface-internal' import type { StubbedInstance } from 'sinon-ts' -const T = MESSAGE_TYPE.FIND_NODE +const T = MessageType.FIND_NODE describe('rpc - handlers - FindNode', () => { let peerId: PeerId @@ -37,12 +39,18 @@ describe('rpc - handlers - FindNode', () => { logger: defaultLogger() }, { peerRouting, - lan: false + logPrefix: '', + peerInfoMapper: passthroughMapper }) }) it('returns self, if asked for self', async () => { - const msg = new Message(T, peerId.multihash.bytes, 0) + const msg: Message = { + type: T, + key: peerId.multihash.bytes, + closer: [], + providers: [] + } addressManager.getAddresses.returns([ multiaddr(`/ip4/127.0.0.1/tcp/4002/p2p/${peerId.toString()}`), @@ -56,14 +64,19 @@ describe('rpc - handlers - FindNode', () => { throw new Error('No response received from handler') } - expect(response.closerPeers).to.have.length(1) - const peer = response.closerPeers[0] + expect(response.closer).to.have.length(1) + const peer = response.closer[0] - expect(peer.id).to.be.eql(peerId) + expect(peerIdFromBytes(peer.id).toString()).to.equal(peerId.toString()) }) it('returns closer peers', async () => { - const msg = new Message(T, targetPeer.multihash.bytes, 0) + const msg: Message = { + type: T, + key: targetPeer.multihash.bytes, + closer: [], + providers: [] + } peerRouting.getCloserPeersOffline .withArgs(targetPeer.multihash.bytes, sourcePeer) @@ -82,25 +95,35 @@ describe('rpc - handlers - FindNode', () => { throw new Error('No response received from handler') } - expect(response.closerPeers).to.have.length(1) - const peer = response.closerPeers[0] + expect(response.closer).to.have.length(1) + const peer = response.closer[0] - expect(peer.id).to.be.eql(targetPeer) + expect(peerIdFromBytes(peer.id).toString()).to.equal(targetPeer.toString()) expect(peer.multiaddrs).to.not.be.empty() }) it('handles no peers found', async () => { - const msg = new Message(T, targetPeer.multihash.bytes, 0) + const msg: Message = { + type: T, + key: targetPeer.multihash.bytes, + closer: [], + providers: [] + } peerRouting.getCloserPeersOffline.resolves([]) const response = await handler.handle(sourcePeer, msg) - expect(response).to.have.property('closerPeers').that.is.empty() + expect(response).to.have.property('closer').that.is.empty() }) it('returns only lan addresses', async () => { - const msg = new Message(T, targetPeer.multihash.bytes, 0) + const msg: Message = { + type: T, + key: targetPeer.multihash.bytes, + closer: [], + providers: [] + } peerRouting.getCloserPeersOffline .withArgs(targetPeer.multihash.bytes, sourcePeer) @@ -119,7 +142,8 @@ describe('rpc - handlers - FindNode', () => { logger: defaultLogger() }, { peerRouting, - lan: true + logPrefix: '', + peerInfoMapper: removePublicAddressesMapper }) const response = await handler.handle(sourcePeer, msg) @@ -128,16 +152,21 @@ describe('rpc - handlers - FindNode', () => { throw new Error('No response received from handler') } - expect(response.closerPeers).to.have.length(1) - const peer = response.closerPeers[0] + expect(response.closer).to.have.length(1) + const peer = response.closer[0] - expect(peer.id).to.be.eql(targetPeer) - expect(peer.multiaddrs.map(ma => ma.toString())).to.include('/ip4/192.168.1.5/tcp/4002') - expect(peer.multiaddrs.map(ma => ma.toString())).to.not.include('/ip4/221.4.67.0/tcp/4002') + expect(peerIdFromBytes(peer.id).toString()).to.equal(targetPeer.toString()) + expect(peer.multiaddrs.map(ma => multiaddr(ma).toString())).to.include('/ip4/192.168.1.5/tcp/4002') + expect(peer.multiaddrs.map(ma => multiaddr(ma).toString())).to.not.include('/ip4/221.4.67.0/tcp/4002') }) it('returns only wan addresses', async () => { - const msg = new Message(T, targetPeer.multihash.bytes, 0) + const msg: Message = { + type: T, + key: targetPeer.multihash.bytes, + closer: [], + providers: [] + } peerRouting.getCloserPeersOffline .withArgs(targetPeer.multihash.bytes, sourcePeer) @@ -150,17 +179,27 @@ describe('rpc - handlers - FindNode', () => { ] }]) + handler = new FindNodeHandler({ + peerId, + addressManager, + logger: defaultLogger() + }, { + peerRouting, + logPrefix: '', + peerInfoMapper: removePrivateAddressesMapper + }) + const response = await handler.handle(sourcePeer, msg) if (response == null) { throw new Error('No response received from handler') } - expect(response.closerPeers).to.have.length(1) - const peer = response.closerPeers[0] + expect(response.closer).to.have.length(1) + const peer = response.closer[0] - expect(peer.id).to.be.eql(targetPeer) - expect(peer.multiaddrs.map(ma => ma.toString())).to.not.include('/ip4/192.168.1.5/tcp/4002') - expect(peer.multiaddrs.map(ma => ma.toString())).to.include('/ip4/221.4.67.0/tcp/4002') + expect(peerIdFromBytes(peer.id).toString()).to.equal(targetPeer.toString()) + expect(peer.multiaddrs.map(ma => multiaddr(ma).toString())).to.not.include('/ip4/192.168.1.5/tcp/4002') + expect(peer.multiaddrs.map(ma => multiaddr(ma).toString())).to.include('/ip4/221.4.67.0/tcp/4002') }) }) diff --git a/packages/kad-dht/test/rpc/handlers/get-providers.spec.ts b/packages/kad-dht/test/rpc/handlers/get-providers.spec.ts index 76df77fc55..a9d486cf43 100644 --- a/packages/kad-dht/test/rpc/handlers/get-providers.spec.ts +++ b/packages/kad-dht/test/rpc/handlers/get-providers.spec.ts @@ -2,21 +2,23 @@ import { TypedEventEmitter } from '@libp2p/interface' import { defaultLogger } from '@libp2p/logger' +import { peerIdFromBytes } from '@libp2p/peer-id' import { PersistentPeerStore } from '@libp2p/peer-store' import { multiaddr } from '@multiformats/multiaddr' import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' import Sinon, { type SinonStubbedInstance } from 'sinon' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Message, MESSAGE_TYPE } from '../../../src/message/index.js' +import { type Message, MessageType } from '../../../src/message/dht.js' import { PeerRouting } from '../../../src/peer-routing/index.js' import { Providers } from '../../../src/providers.js' import { GetProvidersHandler, type GetProvidersHandlerComponents } from '../../../src/rpc/handlers/get-providers.js' +import { passthroughMapper } from '../../../src/utils.js' import { createPeerId } from '../../utils/create-peer-id.js' import { createValues, type Value } from '../../utils/create-values.js' import type { Libp2pEvents, PeerId, PeerInfo, PeerStore } from '@libp2p/interface' -const T = MESSAGE_TYPE.GET_PROVIDERS +const T = MessageType.GET_PROVIDERS describe('rpc - handlers - GetProviders', () => { let peerId: PeerId @@ -53,19 +55,30 @@ describe('rpc - handlers - GetProviders', () => { handler = new GetProvidersHandler(components, { peerRouting, providers, - lan: false + logPrefix: '', + peerInfoMapper: passthroughMapper }) }) it('errors with an invalid key ', async () => { - const msg = new Message(T, uint8ArrayFromString('hello'), 0) + const msg: Message = { + type: T, + key: uint8ArrayFromString('hello'), + closer: [], + providers: [] + } await expect(handler.handle(sourcePeer, msg)).to.eventually.be.rejected().with.property('code', 'ERR_INVALID_CID') }) it('responds with providers and closer peers', async () => { const v = values[0] - const msg = new Message(T, v.cid.bytes, 0) + const msg: Message = { + type: T, + key: v.cid.bytes, + closer: [], + providers: [] + } const closer: PeerInfo[] = [{ id: closerPeer, @@ -102,9 +115,9 @@ describe('rpc - handlers - GetProviders', () => { } expect(response.key).to.be.eql(v.cid.bytes) - expect(response.providerPeers).to.have.lengthOf(1) - expect(response.providerPeers[0].id.toString()).to.equal(provider[0].id.toString()) - expect(response.closerPeers).to.have.lengthOf(1) - expect(response.closerPeers[0].id.toString()).to.equal(closer[0].id.toString()) + expect(response.providers).to.have.lengthOf(1) + expect(peerIdFromBytes(response.providers[0].id).toString()).to.equal(provider[0].id.toString()) + expect(response.closer).to.have.lengthOf(1) + expect(peerIdFromBytes(response.closer[0].id).toString()).to.equal(closer[0].id.toString()) }) }) diff --git a/packages/kad-dht/test/rpc/handlers/get-value.spec.ts b/packages/kad-dht/test/rpc/handlers/get-value.spec.ts index b29d2a975d..f8f03cc57a 100644 --- a/packages/kad-dht/test/rpc/handlers/get-value.spec.ts +++ b/packages/kad-dht/test/rpc/handlers/get-value.spec.ts @@ -7,7 +7,7 @@ import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' import Sinon from 'sinon' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Message, MESSAGE_TYPE } from '../../../src/message/index.js' +import { type Message, MessageType } from '../../../src/message/dht.js' import { PeerRouting } from '../../../src/peer-routing/index.js' import { Libp2pRecord } from '../../../src/record/index.js' import { GetValueHandler, type GetValueHandlerComponents } from '../../../src/rpc/handlers/get-value.js' @@ -17,7 +17,7 @@ import type { Libp2pEvents, PeerId, PeerStore } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' import type { SinonStubbedInstance } from 'sinon' -const T = MESSAGE_TYPE.GET_VALUE +const T = MessageType.GET_VALUE describe('rpc - handlers - GetValue', () => { let peerId: PeerId @@ -50,12 +50,18 @@ describe('rpc - handlers - GetValue', () => { } handler = new GetValueHandler(components, { - peerRouting + peerRouting, + logPrefix: '' }) }) it('errors when missing key', async () => { - const msg = new Message(T, new Uint8Array(0), 0) + const msg: Message = { + type: T, + key: new Uint8Array(0), + closer: [], + providers: [] + } try { await handler.handle(sourcePeer, msg) @@ -74,7 +80,12 @@ describe('rpc - handlers - GetValue', () => { await datastore.put(utils.bufferToRecordKey(key), record.serialize().subarray()) - const msg = new Message(T, key, 0) + const msg: Message = { + type: T, + key, + closer: [], + providers: [] + } peerRouting.getCloserPeersOffline.withArgs(msg.key, sourcePeer).resolves([]) @@ -84,12 +95,16 @@ describe('rpc - handlers - GetValue', () => { throw new Error('No response received from handler') } - expect(response.record).to.exist() - expect(response).to.have.nested.property('record.key').that.equalBytes(key) - expect(response).to.have.nested.property('record.value').that.equalBytes(value) + if (response.record == null) { + throw new Error('No record returned') + } + + const responseRecord = Libp2pRecord.deserialize(response.record) + expect(responseRecord).to.have.property('key').that.equalBytes(key) + expect(responseRecord).to.have.property('value').that.equalBytes(value) }) - it('responds with closerPeers returned from the dht', async () => { + it('responds with closer peers returned from the dht', async () => { const key = uint8ArrayFromString('hello') peerRouting.getCloserPeersOffline.withArgs(key, sourcePeer) @@ -98,20 +113,30 @@ describe('rpc - handlers - GetValue', () => { multiaddrs: [] }]) - const msg = new Message(T, key, 0) + const msg: Message = { + type: T, + key, + closer: [], + providers: [] + } const response = await handler.handle(sourcePeer, msg) if (response == null) { throw new Error('No response received from handler') } - expect(response).to.have.nested.property('closerPeers[0].id').that.deep.equals(closerPeer) + expect(response).to.have.nested.property('closer[0].id').that.deep.equals(closerPeer.toBytes()) }) describe('public key', () => { it('peer in peerstore', async () => { const key = utils.keyForPublicKey(targetPeer) - const msg = new Message(T, key, 0) + const msg: Message = { + type: T, + key, + closer: [], + providers: [] + } if (targetPeer.publicKey == null) { throw new Error('targetPeer had no public key') @@ -127,12 +152,23 @@ describe('rpc - handlers - GetValue', () => { throw new Error('No response received from handler') } - expect(response).to.have.nested.property('record.value').that.equalBytes(targetPeer.publicKey) + if (response.record == null) { + throw new Error('No record returned') + } + + const responseRecord = Libp2pRecord.deserialize(response.record) + + expect(responseRecord).to.have.property('value').that.equalBytes(targetPeer.publicKey) }) it('peer not in peerstore', async () => { const key = utils.keyForPublicKey(targetPeer) - const msg = new Message(T, key, 0) + const msg: Message = { + type: T, + key, + closer: [], + providers: [] + } peerRouting.getCloserPeersOffline.resolves([]) diff --git a/packages/kad-dht/test/rpc/handlers/ping.spec.ts b/packages/kad-dht/test/rpc/handlers/ping.spec.ts index 1cd1a40447..1deef99c8e 100644 --- a/packages/kad-dht/test/rpc/handlers/ping.spec.ts +++ b/packages/kad-dht/test/rpc/handlers/ping.spec.ts @@ -3,13 +3,13 @@ import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Message, MESSAGE_TYPE } from '../../../src/message/index.js' +import { type Message, MessageType } from '../../../src/message/dht.js' import { PingHandler } from '../../../src/rpc/handlers/ping.js' import { createPeerId } from '../../utils/create-peer-id.js' import type { DHTMessageHandler } from '../../../src/rpc/index.js' import type { PeerId } from '@libp2p/interface' -const T = MESSAGE_TYPE.PING +const T = MessageType.PING describe('rpc - handlers - Ping', () => { let sourcePeer: PeerId @@ -22,11 +22,18 @@ describe('rpc - handlers - Ping', () => { beforeEach(async () => { handler = new PingHandler({ logger: defaultLogger() + }, { + logPrefix: '' }) }) it('replies with the same message', async () => { - const msg = new Message(T, uint8ArrayFromString('hello'), 5) + const msg: Message = { + type: T, + key: uint8ArrayFromString('hello'), + closer: [], + providers: [] + } const response = await handler.handle(sourcePeer, msg) expect(response).to.be.deep.equal(msg) diff --git a/packages/kad-dht/test/rpc/handlers/put-value.spec.ts b/packages/kad-dht/test/rpc/handlers/put-value.spec.ts index c747f2432b..b772e9dab7 100644 --- a/packages/kad-dht/test/rpc/handlers/put-value.spec.ts +++ b/packages/kad-dht/test/rpc/handlers/put-value.spec.ts @@ -6,7 +6,7 @@ import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' import delay from 'delay' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Message, MESSAGE_TYPE } from '../../../src/message/index.js' +import { type Message, MessageType } from '../../../src/message/dht.js' import { Libp2pRecord } from '../../../src/record/index.js' import { PutValueHandler } from '../../../src/rpc/handlers/put-value.js' import * as utils from '../../../src/utils.js' @@ -15,7 +15,7 @@ import type { Validators } from '../../../src/index.js' import type { PeerId } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' -const T = MESSAGE_TYPE.PUT_VALUE +const T = MessageType.PUT_VALUE describe('rpc - handlers - PutValue', () => { let sourcePeer: PeerId @@ -34,12 +34,18 @@ describe('rpc - handlers - PutValue', () => { } handler = new PutValueHandler(components, { - validators + validators, + logPrefix: '' }) }) it('errors on missing record', async () => { - const msg = new Message(T, uint8ArrayFromString('hello'), 5) + const msg: Message = { + type: T, + key: uint8ArrayFromString('hello'), + closer: [], + providers: [] + } try { await handler.handle(sourcePeer, msg) @@ -52,13 +58,18 @@ describe('rpc - handlers - PutValue', () => { }) it('stores the record in the datastore', async () => { - const msg = new Message(T, uint8ArrayFromString('/val/hello'), 5) + const msg: Message = { + type: T, + key: uint8ArrayFromString('/val/hello'), + closer: [], + providers: [] + } const record = new Libp2pRecord( uint8ArrayFromString('hello'), uint8ArrayFromString('world'), new Date() ) - msg.record = record + msg.record = record.serialize() validators.val = async () => {} const response = await handler.handle(sourcePeer, msg) diff --git a/packages/kad-dht/test/rpc/index.node.ts b/packages/kad-dht/test/rpc/index.node.ts index 11e08e233d..c4886d22dc 100644 --- a/packages/kad-dht/test/rpc/index.node.ts +++ b/packages/kad-dht/test/rpc/index.node.ts @@ -15,11 +15,12 @@ import Sinon, { type SinonStubbedInstance } from 'sinon' import { stubInterface } from 'sinon-ts' import { Uint8ArrayList } from 'uint8arraylist' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Message, MESSAGE_TYPE } from '../../src/message/index.js' +import { Message, MessageType } from '../../src/message/dht.js' import { PeerRouting } from '../../src/peer-routing/index.js' import { Providers } from '../../src/providers.js' import { RoutingTable } from '../../src/routing-table/index.js' import { RPC, type RPCComponents } from '../../src/rpc/index.js' +import { passthroughMapper } from '../../src/utils.js' import { createPeerId } from '../utils/create-peer-id.js' import type { Validators } from '../../src/index.js' import type { Libp2pEvents, Connection, PeerId, PeerStore } from '@libp2p/interface' @@ -64,25 +65,29 @@ describe('rpc', () => { providers, peerRouting, validators, - lan: false + logPrefix: '', + peerInfoMapper: passthroughMapper }) }) it('calls back with the response', async () => { const defer = pDefer() - const msg = new Message(MESSAGE_TYPE.GET_VALUE, uint8ArrayFromString('hello'), 5) + const msg: Partial = { + type: MessageType.GET_VALUE, + key: uint8ArrayFromString('hello') + } const validateMessage = (res: Uint8ArrayList[]): void => { - const msg = Message.deserialize(res[0]) + const msg = Message.decode(res[0]) expect(msg).to.have.property('key').eql(uint8ArrayFromString('hello')) - expect(msg).to.have.property('closerPeers').eql([]) + expect(msg).to.have.property('closer').eql([]) defer.resolve() } peerRouting.getCloserPeersOffline.resolves([]) const source = pipe( - [msg.serialize()], + [Message.encode(msg)], (source) => lp.encode(source), source => map(source, arr => new Uint8ArrayList(arr)), (source) => all(source) diff --git a/packages/kad-dht/test/utils/test-dht.ts b/packages/kad-dht/test/utils/test-dht.ts index 742eae1357..839078edda 100644 --- a/packages/kad-dht/test/utils/test-dht.ts +++ b/packages/kad-dht/test/utils/test-dht.ts @@ -7,21 +7,21 @@ import { MemoryDatastore } from 'datastore-core/memory' import delay from 'delay' import pRetry from 'p-retry' import { stubInterface } from 'sinon-ts' -import { DefaultDualKadDHT } from '../../src/dual-kad-dht.js' +import { PROTOCOL } from '../../src/constants.js' +import { type KadDHT, type KadDHTComponents, type KadDHTInit } from '../../src/index.js' +import { KadDHT as KadDHTClass } from '../../src/kad-dht.js' import { createPeerId } from './create-peer-id.js' -import type { DualKadDHT, KadDHTComponents, KadDHTInit } from '../../src/index.js' -import type { DefaultKadDHT } from '../../src/kad-dht.js' import type { Libp2pEvents, PeerId, PeerStore } from '@libp2p/interface' import type { AddressManager, ConnectionManager, Registrar } from '@libp2p/interface-internal' export class TestDHT { - private readonly peers: Map + private readonly peers: Map constructor () { this.peers = new Map() } - async spawn (options: Partial = {}, autoStart = true): Promise { + async spawn (options: Partial = {}, autoStart = true): Promise { const events = new TypedEventEmitter() const components: KadDHTComponents = { peerId: await createPeerId(), @@ -74,10 +74,11 @@ export class TestDHT { querySelfInterval: 600000, initialQuerySelfInterval: 600000, allowQueryWithZeroPeers: true, + clientMode: false, ...options } - const dht = new DefaultDualKadDHT(components, opts) + const dht = new KadDHTClass(components, opts) // simulate libp2p._onDiscoveryPeer dht.addEventListener('peer', (evt) => { @@ -93,7 +94,7 @@ export class TestDHT { }) if (autoStart) { - await dht.start() + await start(dht) } this.peers.set(components.peerId.toString(), { @@ -104,9 +105,7 @@ export class TestDHT { return dht } - async connect (dhtA: DefaultDualKadDHT, dhtB: DefaultDualKadDHT): Promise { - // need addresses in the address book otherwise we won't know whether to add - // the peer to the public or private DHT and will do nothing + async connect (dhtA: KadDHTClass, dhtB: KadDHTClass): Promise { await dhtA.components.peerStore.merge(dhtB.components.peerId, { multiaddrs: dhtB.components.addressManager.getAddresses() }) @@ -114,40 +113,48 @@ export class TestDHT { multiaddrs: dhtA.components.addressManager.getAddresses() }) - await dhtA.components.connectionManager.openConnection(dhtB.components.peerId) + const connection = await dhtA.components.connectionManager.openConnection(dhtB.components.peerId) - // wait for peers to appear in each others' routing tables - await checkConnected(dhtA.lan, dhtB.lan) + // simulate identify + dhtA.components.registrar.getTopologies(PROTOCOL).forEach(topology => { + topology.onConnect?.(dhtB.components.peerId, connection) + }) + dhtB.components.registrar.getTopologies(PROTOCOL).forEach(topology => { + const conn = dhtB.components.connectionManager.getConnections(dhtA.components.peerId) + topology.onConnect?.(dhtA.components.peerId, conn[0]) + }) - // only wait for WANs to connect if we are in server mode - if ((await dhtA.wan.getMode()) === 'server' && (await dhtB.wan.getMode()) === 'server') { - await checkConnected(dhtA.wan, dhtB.wan) - } + // wait for peers to appear in each others' routing tables + await checkConnected(dhtA, dhtB) - async function checkConnected (a: DefaultKadDHT, b: DefaultKadDHT): Promise { - const routingTableChecks = [] + async function checkConnected (a: KadDHTClass, b: KadDHTClass): Promise { + const routingTableChecks: Array<() => Promise> = [] - routingTableChecks.push(async () => { - const match = await a.routingTable.find(dhtB.components.peerId) + if (b.getMode() === 'server') { + routingTableChecks.push(async () => { + const match = await a.routingTable.find(b.components.peerId) - if (match == null) { - await delay(100) - throw new Error('not found') - } + if (match == null) { + await delay(100) + throw new Error('not found') + } - return match - }) + return match + }) + } - routingTableChecks.push(async () => { - const match = await b.routingTable.find(dhtA.components.peerId) + if (a.getMode() === 'server') { + routingTableChecks.push(async () => { + const match = await b.routingTable.find(a.components.peerId) - if (match == null) { - await delay(100) - throw new Error('not found') - } + if (match == null) { + await delay(100) + throw new Error('not found') + } - return match - }) + return match + }) + } // Check routing tables return Promise.all(