fix: replace node buffers with uint8arrays (#202)

* fix: replace node buffers with uint8arrays Replaces all use of node Buffers with Uint8Arrays BREAKING CHANGES: - Where node Buffers were returned, now Uint8Arrays are * chore: remove gh dep urls
libp2p · Aug 10, 2020 · 989be87 · 989be87
1 parent 8eabccf
commit 989be87
Show file tree

Hide file tree

Showing 34 changed files with 248 additions and 244 deletions.
diff --git a/package.json b/package.json
@@ -40,39 +40,39 @@
     "abort-controller": "^3.0.0",
     "async": "^2.6.2",
     "base32.js": "~0.1.0",
-    "buffer": "^5.6.0",
-    "cids": "~0.8.0",
+    "cids": "^1.0.0",
     "debug": "^4.1.1",
     "err-code": "^2.0.0",
     "hashlru": "^2.3.0",
     "heap": "~0.2.6",
-    "interface-datastore": "^1.0.2",
+    "interface-datastore": "^2.0.0",
     "it-length-prefixed": "^3.0.0",
     "it-pipe": "^1.1.0",
     "k-bucket": "^5.0.0",
-    "libp2p-crypto": "~0.17.1",
-    "libp2p-interfaces": "^0.3.0",
-    "libp2p-record": "~0.7.0",
-    "multiaddr": "^7.4.3",
-    "multihashing-async": "^0.8.2",
+    "libp2p-crypto": "^0.18.0",
+    "libp2p-interfaces": "^0.3.2",
+    "libp2p-record": "^0.9.0",
+    "multiaddr": "^8.0.0",
+    "multihashing-async": "^2.0.1",
     "p-filter": "^2.1.0",
     "p-map": "^4.0.0",
     "p-queue": "^6.2.1",
     "p-timeout": "^3.2.0",
     "p-times": "^3.0.0",
-    "peer-id": "~0.13.5",
+    "peer-id": "^0.14.0",
     "promise-to-callback": "^1.0.0",
-    "protons": "^1.0.1",
-    "streaming-iterables": "^4.1.1",
+    "protons": "^2.0.0",
+    "streaming-iterables": "^5.0.2",
+    "uint8arrays": "^1.1.0",
     "varint": "^5.0.0",
     "xor-distance": "^2.0.0"
   },
   "devDependencies": {
-    "aegir": "^23.0.0",
+    "aegir": "^25.0.0",
     "async-iterator-all": "^1.0.0",
     "chai": "^4.2.0",
     "chai-checkmark": "^1.0.1",
-    "datastore-level": "^1.1.0",
+    "datastore-level": "^2.0.0",
     "delay": "^4.3.0",
     "dirty-chai": "^2.0.1",
     "it-pair": "^1.0.0",

diff --git a/src/content-fetching/index.js b/src/content-fetching/index.js
@@ -2,7 +2,7 @@
 
 const errcode = require('err-code')
 const pTimeout = require('p-timeout')
-
+const uint8ArrayEquals = require('uint8arrays/equals')
 const libp2pRecord = require('libp2p-record')
 
 const c = require('../constants')
@@ -21,7 +21,7 @@ module.exports = (dht) => {
    * Attempt to retrieve the value for the given key from
    * the local datastore.
    *
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @returns {Promise<Record>}
    *
    * @private
@@ -40,7 +40,7 @@ module.exports = (dht) => {
   /**
    * Send the best record found to any peers that have an out of date record.
    *
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @param {Array<Object>} vals - values retrieved from the DHT
    * @param {Object} best - the best record that was found
    * @returns {Promise}
@@ -52,7 +52,7 @@ module.exports = (dht) => {
 
     return Promise.all(vals.map(async (v) => {
       // no need to do anything
-      if (v.val.equals(best)) {
+      if (uint8ArrayEquals(v.val, best)) {
         return
       }
 
@@ -78,8 +78,8 @@ module.exports = (dht) => {
   return {
     /**
      * Store the given key/value pair locally, in the datastore.
-     * @param {Buffer} key
-     * @param {Buffer} rec - encoded record
+     * @param {Uint8Array} key
+     * @param {Uint8Array} rec - encoded record
      * @returns {Promise<void>}
      * @private
      */
@@ -90,8 +90,8 @@ module.exports = (dht) => {
     /**
      * Store the given key/value  pair in the DHT.
      *
-     * @param {Buffer} key
-     * @param {Buffer} value
+     * @param {Uint8Array} key
+     * @param {Uint8Array} value
      * @param {Object} [options] - put options
      * @param {number} [options.minPeers] - minimum number of peers required to successfully put (default: closestPeers.length)
      * @returns {Promise<void>}
@@ -133,10 +133,10 @@ module.exports = (dht) => {
      * Get the value to the given key.
      * Times out after 1 minute by default.
      *
-     * @param {Buffer} key
+     * @param {Uint8Array} key
      * @param {Object} [options] - get options
      * @param {number} [options.timeout] - optional timeout (default: 60000)
-     * @returns {Promise<Buffer>}
+     * @returns {Promise<Uint8Array>}
      */
     async get (key, options = {}) {
       options.timeout = options.timeout || c.minute
@@ -171,11 +171,11 @@ module.exports = (dht) => {
     /**
      * Get the `n` values to the given key without sorting.
      *
-     * @param {Buffer} key
+     * @param {Uint8Array} key
      * @param {number} nvals
      * @param {Object} [options] - get options
      * @param {number} [options.timeout] - optional timeout (default: 60000)
-     * @returns {Promise<Array<{from: PeerId, val: Buffer}>>}
+     * @returns {Promise<Array<{from: PeerId, val: Uint8Array}>>}
      */
     async getMany (key, nvals, options = {}) {
       options.timeout = options.timeout || c.minute

diff --git a/src/content-routing/index.js b/src/content-routing/index.js
@@ -20,7 +20,7 @@ module.exports = (dht) => {
    * @private
    */
   const findProvidersSingle = async (peer, key) => { // eslint-disable-line require-await
-    const msg = new Message(Message.TYPES.GET_PROVIDERS, key.buffer, 0)
+    const msg = new Message(Message.TYPES.GET_PROVIDERS, key.bytes, 0)
     return dht.network.sendRequest(peer, msg)
   }
 
@@ -39,14 +39,14 @@ module.exports = (dht) => {
       await dht.providers.addProvider(key, dht.peerId)
 
       const multiaddrs = dht.libp2p ? dht.libp2p.multiaddrs : []
-      const msg = new Message(Message.TYPES.ADD_PROVIDER, key.buffer, 0)
+      const msg = new Message(Message.TYPES.ADD_PROVIDER, key.bytes, 0)
       msg.providerPeers = [{
         id: dht.peerId,
         multiaddrs
       }]
 
       // Notify closest peers
-      await utils.mapParallel(dht.getClosestPeers(key.buffer), async (peer) => {
+      await utils.mapParallel(dht.getClosestPeers(key.bytes), async (peer) => {
         dht._log('putProvider %s to %s', key.toBaseEncodedString(), peer.toB58String())
         try {
           await dht.network.sendMessage(peer, msg)
@@ -100,7 +100,7 @@ module.exports = (dht) => {
 
       // need more, query the network
       const paths = []
-      const query = new Query(dht, key.buffer, (pathIndex, numPaths) => {
+      const query = new Query(dht, key.bytes, (pathIndex, numPaths) => {
         // This function body runs once per disjoint path
         const pathSize = utils.pathSize(n - out.length, numPaths)
         const pathProviders = new LimitedPeerList(pathSize)
@@ -126,7 +126,7 @@ module.exports = (dht) => {
         }
       })
 
-      const peers = dht.routingTable.closestPeers(key.buffer, dht.kBucketSize)
+      const peers = dht.routingTable.closestPeers(key.bytes, dht.kBucketSize)
 
       try {
         await pTimeout(

diff --git a/src/index.js b/src/index.js
@@ -5,6 +5,7 @@ const errcode = require('err-code')
 
 const libp2pRecord = require('libp2p-record')
 const { MemoryDatastore } = require('interface-datastore')
+const uint8ArrayEquals = require('uint8arrays/equals')
 
 const RoutingTable = require('./routing')
 const utils = require('./utils')
@@ -220,8 +221,8 @@ class KadDHT extends EventEmitter {
 
   /**
    * Store the given key/value  pair in the DHT.
-   * @param {Buffer} key
-   * @param {Buffer} value
+   * @param {Uint8Array} key
+   * @param {Uint8Array} value
    * @param {Object} [options] - put options
    * @param {number} [options.minPeers] - minimum number of peers required to successfully put (default: closestPeers.length)
    * @returns {Promise<void>}
@@ -233,22 +234,22 @@ class KadDHT extends EventEmitter {
   /**
    * Get the value to the given key.
    * Times out after 1 minute by default.
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @param {Object} [options] - get options
    * @param {number} [options.timeout] - optional timeout (default: 60000)
-   * @returns {Promise<Buffer>}
+   * @returns {Promise<Uint8Array>}
    */
   async get (key, options = {}) { // eslint-disable-line require-await
     return this.contentFetching.get(key, options)
   }
 
   /**
    * Get the `n` values to the given key without sorting.
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @param {number} nvals
    * @param {Object} [options] - get options
    * @param {number} [options.timeout] - optional timeout (default: 60000)
-   * @returns {Promise<Array<{from: PeerId, val: Buffer}>>}
+   * @returns {Promise<Array<{from: PeerId, val: Uint8Array}>>}
    */
   async getMany (key, nvals, options = {}) { // eslint-disable-line require-await
     return this.contentFetching.getMany(key, nvals, options)
@@ -295,7 +296,7 @@ class KadDHT extends EventEmitter {
 
   /**
    * Kademlia 'node lookup' operation.
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @param {Object} [options]
    * @param {boolean} [options.shallow] shallow query (default: false)
    * @returns {AsyncIterable<{ id: PeerId, multiaddrs: Multiaddr[] }>}
@@ -378,7 +379,7 @@ class KadDHT extends EventEmitter {
    * - it was either authored by this node, or
    * - it was received less than `MAX_RECORD_AGE` ago.
    *
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @returns {Promise<Record>}
    * @private
    */
@@ -451,14 +452,14 @@ class KadDHT extends EventEmitter {
    */
 
   _isSelf (other) {
-    return other && this.peerId.id.equals(other.id)
+    return other && uint8ArrayEquals(this.peerId.id, other.id)
   }
 
   /**
    * Store the given key/value pair at the peer `target`.
    *
-   * @param {Buffer} key
-   * @param {Buffer} rec - encoded record
+   * @param {Uint8Array} key
+   * @param {Uint8Array} rec - encoded record
    * @param {PeerId} target
    * @returns {Promise<void>}
    *
@@ -483,7 +484,7 @@ class KadDHT extends EventEmitter {
    * Note: The peerStore is updated with new addresses found for the given peer.
    *
    * @param {PeerId} peer
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @returns {Promise<{Record, Array<{ id: PeerId, multiaddrs: Multiaddr[] }}>}
    * @private
    */
@@ -518,7 +519,7 @@ class KadDHT extends EventEmitter {
    * Get a value via rpc call for the given parameters.
    *
    * @param {PeerId} peer
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @returns {Promise<Message>}
    * @private
    */

diff --git a/src/message/index.js b/src/message/index.js
@@ -4,7 +4,6 @@ const PeerId = require('peer-id')
 const multiaddr = require('multiaddr')
 const protons = require('protons')
 const { Record } = require('libp2p-record')
-const { Buffer } = require('buffer')
 const pbm = protons(require('./dht.proto'))
 
 const MESSAGE_TYPE = pbm.Message.MessageType
@@ -16,12 +15,12 @@ const CONNECTION_TYPE = pbm.Message.ConnectionType
 class Message {
   /**
    * @param {MessageType} type
-   * @param {Buffer} key
+   * @param {Uint8Array} key
    * @param {number} level
    */
   constructor (type, key, level) {
-    if (key && !Buffer.isBuffer(key)) {
-      throw new Error('Key must be a buffer')
+    if (key && !(key instanceof Uint8Array)) {
+      throw new Error('Key must be a Uint8Array')
     }
 
     this.type = type
@@ -50,7 +49,7 @@ class Message {
 
   /**
    * Encode into protobuf
-   * @returns {Buffer}
+   * @returns {Uint8Array}
    */
   serialize () {
     const obj = {
@@ -62,7 +61,7 @@ class Message {
     }
 
     if (this.record) {
-      if (Buffer.isBuffer(this.record)) {
+      if (this.record instanceof Uint8Array) {
         obj.record = this.record
       } else {
         obj.record = this.record.serialize()
@@ -75,7 +74,7 @@ class Message {
   /**
    * Decode from protobuf
    *
-   * @param {Buffer} raw
+   * @param {Uint8Array} raw
    * @returns {Message}
    */
   static deserialize (raw) {
@@ -99,7 +98,7 @@ Message.CONNECTION_TYPES = CONNECTION_TYPE
 function toPbPeer (peer) {
   return {
     id: peer.id.id,
-    addrs: (peer.multiaddrs || []).map((m) => m.buffer),
+    addrs: (peer.multiaddrs || []).map((m) => m.bytes),
     connection: CONNECTION_TYPE.CONNECTED
   }
 }

diff --git a/src/network.js b/src/network.js
@@ -164,7 +164,7 @@ class Network {
    * this will error out.
    *
    * @param {DuplexIterable} stream - the stream to use
-   * @param {Buffer} msg - the message to send
+   * @param {Uint8Array} msg - the message to send
    * @returns {Promise<Message>}
    * @private
    */
@@ -179,7 +179,7 @@ class Network {
    * Write a message to the given stream.
    *
    * @param {DuplexIterable} stream - the stream to use
-   * @param {Buffer} msg - the message to send
+   * @param {Uint8Array} msg - the message to send
    * @returns {Promise<void>}
    * @private
    */

diff --git a/src/peer-list/peer-distance-list.js b/src/peer-list/peer-distance-list.js
@@ -3,6 +3,7 @@
 const distance = require('xor-distance')
 const utils = require('../utils')
 const pMap = require('p-map')
+const uint8ArrayEquals = require('uint8arrays/equals')
 
 /**
  * Maintains a list of peerIds sorted by distance from a DHT key.
@@ -11,7 +12,7 @@ class PeerDistanceList {
   /**
    * Creates a new PeerDistanceList.
    *
-   * @param {Buffer} originDhtKey - the DHT key from which distance is calculated
+   * @param {Uint8Array} originDhtKey - the DHT key from which distance is calculated
    * @param {number} capacity - the maximum size of the list
    */
   constructor (originDhtKey, capacity) {
@@ -41,7 +42,7 @@ class PeerDistanceList {
    * @returns {Promise<void>}
    */
   async add (peerId) {
-    if (this.peerDistances.find(pd => pd.peerId.id.equals(peerId.id))) {
+    if (this.peerDistances.find(pd => uint8ArrayEquals(pd.peerId.id, peerId.id))) {
       return
     }