From 6a62d1c8dcfadead0498d0bb59958837dc204c91 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 26 Apr 2024 05:43:23 +0100 Subject: [PATCH] fix: http blockbroker loads gateways from routing (#519) Use the new http gateway block broker to load gateways from the routing. If we can find http gateways for a block we now use it, falling back to the default list of gateways if the routing is slow or it gives providers that fail to supply the block. One change here is that unless you are using sessions, trustless gateways become disposable so we can't sort them by reliability any more. --- packages/block-brokers/.aegir.js | 43 ++++- packages/block-brokers/package.json | 3 +- .../src/trustless-gateway/broker.ts | 33 ++-- .../src/trustless-gateway/index.ts | 28 +-- .../src/trustless-gateway/session.ts | 47 +---- .../src/trustless-gateway/utils.ts | 45 +++++ .../test/fixtures/create-block.ts | 18 -- .../test/trustless-gateway.spec.ts | 169 +++++++----------- packages/helia/src/index.ts | 5 +- packages/http/src/index.ts | 6 +- packages/routers/src/http-gateway-routing.ts | 12 +- 11 files changed, 191 insertions(+), 218 deletions(-) create mode 100644 packages/block-brokers/src/trustless-gateway/utils.ts delete mode 100644 packages/block-brokers/test/fixtures/create-block.ts diff --git a/packages/block-brokers/.aegir.js b/packages/block-brokers/.aegir.js index 35d044d85..183c9b308 100644 --- a/packages/block-brokers/.aegir.js +++ b/packages/block-brokers/.aegir.js @@ -5,19 +5,19 @@ import polka from 'polka' const options = { test: { async before (options) { - const server = polka({ + const goodGateway = polka({ port: 0, host: '127.0.0.1' }) - server.use(cors()) - server.all('/ipfs/bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq', (req, res) => { + goodGateway.use(cors()) + goodGateway.all('/ipfs/bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq', (req, res) => { res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-length': 4 }) res.end(Uint8Array.from([0, 1, 2, 0])) }) - server.all('/ipfs/bafkqabtimvwgy3yk', async (req, res) => { + goodGateway.all('/ipfs/bafkqabtimvwgy3yk', async (req, res) => { // delay the response await new Promise((resolve) => setTimeout(resolve, 500)) @@ -29,18 +29,43 @@ const options = { res.end(Uint8Array.from([104, 101, 108, 108, 111])) }) - await server.listen() - const { port } = server.server.address() + await goodGateway.listen() + const { port: goodGatewayPort } = goodGateway.server.address() + + const badGateway = polka({ + port: 0, + host: '127.0.0.1' + }) + badGateway.use(cors()) + badGateway.all('/ipfs/bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq', (req, res) => { + res.writeHead(200, { + 'content-type': 'application/octet-stream', + 'content-length': 4 + }) + // fails validation + res.end(Uint8Array.from([0, 1, 2, 1])) + }) + badGateway.all('/ipfs/*', (req, res) => { + // fails + res.writeHead(500) + res.end() + }) + + await badGateway.listen() + const { port: badGatewayPort } = badGateway.server.address() return { - server, + goodGateway, + badGateway, env: { - TRUSTLESS_GATEWAY: `http://127.0.0.1:${port}` + TRUSTLESS_GATEWAY: `http://127.0.0.1:${goodGatewayPort}`, + BAD_TRUSTLESS_GATEWAY: `http://127.0.0.1:${badGatewayPort}` } } }, async after (options, before) { - await before.server.server.close() + await before.goodGateway.server.close() + await before.badGateway.server.close() } } } diff --git a/packages/block-brokers/package.json b/packages/block-brokers/package.json index 003ce99bb..d8743fd4c 100644 --- a/packages/block-brokers/package.json +++ b/packages/block-brokers/package.json @@ -76,6 +76,7 @@ "cors": "^2.8.5", "polka": "^0.5.2", "sinon": "^17.0.1", - "sinon-ts": "^2.0.0" + "sinon-ts": "^2.0.0", + "uint8arrays": "^5.0.3" } } diff --git a/packages/block-brokers/src/trustless-gateway/broker.ts b/packages/block-brokers/src/trustless-gateway/broker.ts index 83210d719..ec566a90f 100644 --- a/packages/block-brokers/src/trustless-gateway/broker.ts +++ b/packages/block-brokers/src/trustless-gateway/broker.ts @@ -1,6 +1,6 @@ import { createTrustlessGatewaySession } from './session.js' -import { TrustlessGateway } from './trustless-gateway.js' -import { DEFAULT_TRUSTLESS_GATEWAYS } from './index.js' +import { findHttpGatewayProviders } from './utils.js' +import { DEFAULT_ALLOW_INSECURE, DEFAULT_ALLOW_LOCAL } from './index.js' import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayComponents, TrustlessGatewayGetBlockProgressEvents } from './index.js' import type { Routing, BlockRetrievalOptions, BlockBroker, CreateSessionOptions } from '@helia/interface' import type { ComponentLogger, Logger } from '@libp2p/interface' @@ -29,55 +29,48 @@ export interface CreateTrustlessGatewaySessionOptions extends CreateSessionOptio * for blocks. */ export class TrustlessGatewayBlockBroker implements BlockBroker { - private readonly components: TrustlessGatewayComponents - private readonly gateways: TrustlessGateway[] + private readonly allowInsecure: boolean + private readonly allowLocal: boolean private readonly routing: Routing private readonly log: Logger private readonly logger: ComponentLogger constructor (components: TrustlessGatewayComponents, init: TrustlessGatewayBlockBrokerInit = {}) { - this.components = components this.log = components.logger.forComponent('helia:trustless-gateway-block-broker') this.logger = components.logger this.routing = components.routing - this.gateways = (init.gateways ?? DEFAULT_TRUSTLESS_GATEWAYS) - .map((gatewayOrUrl) => { - return new TrustlessGateway(gatewayOrUrl, components.logger) - }) - } - - addGateway (gatewayOrUrl: string): void { - this.gateways.push(new TrustlessGateway(gatewayOrUrl, this.components.logger)) + this.allowInsecure = init.allowInsecure ?? DEFAULT_ALLOW_INSECURE + this.allowLocal = init.allowLocal ?? DEFAULT_ALLOW_LOCAL } async retrieve (cid: CID, options: BlockRetrievalOptions = {}): Promise { - // Loop through the gateways until we get a block or run out of gateways - // TODO: switch to toSorted when support is better - const sortedGateways = this.gateways.sort((a, b) => b.reliability() - a.reliability()) const aggregateErrors: Error[] = [] - for (const gateway of sortedGateways) { + for await (const gateway of findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, options)) { this.log('getting block for %c from %s', cid, gateway.url) + try { const block = await gateway.getRawBlock(cid, options.signal) this.log.trace('got block for %c from %s', cid, gateway.url) + try { await options.validateFn?.(block) } catch (err) { this.log.error('failed to validate block for %c from %s', cid, gateway.url, err) - gateway.incrementInvalidBlocks() - - throw new Error(`Block for CID ${cid} from gateway ${gateway.url} failed validation`) + // try another gateway + continue } return block } catch (err: unknown) { this.log.error('failed to get block for %c from %s', cid, gateway.url, err) + if (err instanceof Error) { aggregateErrors.push(err) } else { aggregateErrors.push(new Error(`Unable to fetch raw block for CID ${cid} from gateway ${gateway.url}`)) } + // if signal was aborted, exit the loop if (options.signal?.aborted === true) { this.log.trace('request aborted while fetching raw block for CID %c from gateway %s', cid, gateway.url) diff --git a/packages/block-brokers/src/trustless-gateway/index.ts b/packages/block-brokers/src/trustless-gateway/index.ts index 16f0ac07d..f8627625c 100644 --- a/packages/block-brokers/src/trustless-gateway/index.ts +++ b/packages/block-brokers/src/trustless-gateway/index.ts @@ -3,22 +3,28 @@ import type { Routing, BlockBroker } from '@helia/interface' import type { ComponentLogger } from '@libp2p/interface' import type { ProgressEvent } from 'progress-events' -export const DEFAULT_TRUSTLESS_GATEWAYS = [ - // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/ - 'https://trustless-gateway.link', - - // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/ - 'https://cloudflare-ipfs.com', - - // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/ - 'https://4everland.io' -] +export const DEFAULT_ALLOW_INSECURE = false +export const DEFAULT_ALLOW_LOCAL = false export type TrustlessGatewayGetBlockProgressEvents = ProgressEvent<'trustless-gateway:get-block:fetch', URL> export interface TrustlessGatewayBlockBrokerInit { - gateways?: Array + /** + * By default we will only connect to peers with HTTPS addresses, pass true + * to also connect to HTTP addresses. + * + * @default false + */ + allowInsecure?: boolean + + /** + * By default we will only connect to peers with public or DNS addresses, pass + * true to also connect to private addresses. + * + * @default false + */ + allowLocal?: boolean } export interface TrustlessGatewayComponents { diff --git a/packages/block-brokers/src/trustless-gateway/session.ts b/packages/block-brokers/src/trustless-gateway/session.ts index e86818c2d..9d26d7435 100644 --- a/packages/block-brokers/src/trustless-gateway/session.ts +++ b/packages/block-brokers/src/trustless-gateway/session.ts @@ -1,19 +1,14 @@ import { AbstractSession } from '@helia/utils' -import { isPrivateIp } from '@libp2p/utils/private-ip' -import { DNS, HTTP, HTTPS } from '@multiformats/multiaddr-matcher' -import { multiaddrToUri } from '@multiformats/multiaddr-to-uri' -import { TrustlessGateway } from './trustless-gateway.js' +import { findHttpGatewayProviders } from './utils.js' +import { DEFAULT_ALLOW_INSECURE, DEFAULT_ALLOW_LOCAL } from './index.js' import type { CreateTrustlessGatewaySessionOptions } from './broker.js' import type { TrustlessGatewayGetBlockProgressEvents } from './index.js' +import type { TrustlessGateway } from './trustless-gateway.js' import type { BlockRetrievalOptions, Routing } from '@helia/interface' import type { ComponentLogger } from '@libp2p/interface' -import type { Multiaddr } from '@multiformats/multiaddr' import type { AbortOptions } from 'interface-store' import type { CID } from 'multiformats/cid' -const DEFAULT_ALLOW_INSECURE = false -const DEFAULT_ALLOW_LOCAL = false - export interface TrustlessGatewaySessionComponents { logger: ComponentLogger routing: Routing @@ -47,23 +42,7 @@ class TrustlessGatewaySession extends AbstractSession { - for await (const provider of this.routing.findProviders(cid, options)) { - // require http(s) addresses - const httpAddresses = filterMultiaddrs(provider.multiaddrs, this.allowInsecure, this.allowLocal) - - if (httpAddresses.length === 0) { - continue - } - - // take first address? - // /ip4/x.x.x.x/tcp/31337/http - // /ip4/x.x.x.x/tcp/31337/https - // etc - const uri = multiaddrToUri(httpAddresses[0]) - - this.log('found http-gateway provider %p %s for cid %c', provider.id, uri, cid) - yield new TrustlessGateway(uri, this.logger) - } + yield * findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, options) } toEvictionKey (provider: TrustlessGateway): Uint8Array | string { @@ -75,24 +54,6 @@ class TrustlessGatewaySession extends AbstractSession { - if (HTTPS.matches(ma) || (allowInsecure && HTTP.matches(ma))) { - if (allowLocal) { - return true - } - - if (DNS.matches(ma)) { - return true - } - - return isPrivateIp(ma.toOptions().host) === false - } - - return false - }) -} - export function createTrustlessGatewaySession (components: TrustlessGatewaySessionComponents, init: CreateTrustlessGatewaySessionOptions): TrustlessGatewaySession { return new TrustlessGatewaySession(components, init) } diff --git a/packages/block-brokers/src/trustless-gateway/utils.ts b/packages/block-brokers/src/trustless-gateway/utils.ts new file mode 100644 index 000000000..52aa9ba02 --- /dev/null +++ b/packages/block-brokers/src/trustless-gateway/utils.ts @@ -0,0 +1,45 @@ +import { isPrivateIp } from '@libp2p/utils/private-ip' +import { DNS, HTTP, HTTPS } from '@multiformats/multiaddr-matcher' +import { multiaddrToUri } from '@multiformats/multiaddr-to-uri' +import { TrustlessGateway } from './trustless-gateway.js' +import type { Routing } from '@helia/interface' +import type { ComponentLogger } from '@libp2p/interface' +import type { AbortOptions, Multiaddr } from '@multiformats/multiaddr' +import type { CID } from 'multiformats/cid' + +export function filterNonHTTPMultiaddrs (multiaddrs: Multiaddr[], allowInsecure: boolean, allowLocal: boolean): Multiaddr[] { + return multiaddrs.filter(ma => { + if (HTTPS.matches(ma) || (allowInsecure && HTTP.matches(ma))) { + if (allowLocal) { + return true + } + + if (DNS.matches(ma)) { + return true + } + + return isPrivateIp(ma.toOptions().host) === false + } + + return false + }) +} + +export async function * findHttpGatewayProviders (cid: CID, routing: Routing, logger: ComponentLogger, allowInsecure: boolean, allowLocal: boolean, options?: AbortOptions): AsyncGenerator { + for await (const provider of routing.findProviders(cid, options)) { + // require http(s) addresses + const httpAddresses = filterNonHTTPMultiaddrs(provider.multiaddrs, allowInsecure, allowLocal) + + if (httpAddresses.length === 0) { + continue + } + + // take first address? + // /ip4/x.x.x.x/tcp/31337/http + // /ip4/x.x.x.x/tcp/31337/https + // etc + const uri = multiaddrToUri(httpAddresses[0]) + + yield new TrustlessGateway(uri, logger) + } +} diff --git a/packages/block-brokers/test/fixtures/create-block.ts b/packages/block-brokers/test/fixtures/create-block.ts deleted file mode 100644 index bec0ae374..000000000 --- a/packages/block-brokers/test/fixtures/create-block.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { CID } from 'multiformats/cid' -import { sha256 } from 'multiformats/hashes/sha2' -import type { Blockstore } from 'interface-blockstore' - -export async function createBlock (codec: Codec, block: Uint8Array): Promise<{ cid: CID, block: Uint8Array }> { - const mh = await sha256.digest(block) - const cid = CID.createV1(codec, mh) - - return { cid, block } -} - -export async function createAndPutBlock (codec: Codec, block: Uint8Array, blockstore: Blockstore): Promise> { - const result = await createBlock(codec, block) - - await blockstore.put(result.cid, block) - - return result.cid -} diff --git a/packages/block-brokers/test/trustless-gateway.spec.ts b/packages/block-brokers/test/trustless-gateway.spec.ts index b36cff030..3e8aefdf4 100644 --- a/packages/block-brokers/test/trustless-gateway.spec.ts +++ b/packages/block-brokers/test/trustless-gateway.spec.ts @@ -6,155 +6,106 @@ import { multiaddr } from '@multiformats/multiaddr' import { uriToMultiaddr } from '@multiformats/uri-to-multiaddr' import { expect } from 'aegir/chai' import { CID } from 'multiformats/cid' -import * as raw from 'multiformats/codecs/raw' -import Sinon from 'sinon' -import { type StubbedInstance, stubConstructor, stubInterface } from 'sinon-ts' +import { sha256 } from 'multiformats/hashes/sha2' +import { stubInterface } from 'sinon-ts' +import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import { TrustlessGatewayBlockBroker } from '../src/trustless-gateway/broker.js' import { TrustlessGateway } from '../src/trustless-gateway/trustless-gateway.js' -import { createBlock } from './fixtures/create-block.js' import type { Routing } from '@helia/interface' +import type { PeerInfo } from '@libp2p/interface' +import type { StubbedInstance } from 'sinon-ts' describe('trustless-gateway-block-broker', () => { - let blocks: Array<{ cid: CID, block: Uint8Array }> let gatewayBlockBroker: TrustlessGatewayBlockBroker - let gateways: Array> let routing: StubbedInstance - - // take a Record) => void> and stub the gateways - // Record.default is the default handler - function stubGateways (handlers: Record, index?: number) => void> & { default(gateway: StubbedInstance, index: number): void }): void { - for (let i = 0; i < gateways.length; i++) { - if (handlers[i] != null) { - handlers[i](gateways[i]) - continue - } - handlers.default(gateways[i], i) - } - } + let badGatewayPeer: PeerInfo + let goodGatewayPeer: PeerInfo + let cid: CID + let expectedBlock: Uint8Array beforeEach(async () => { + // see .aegir.js - bad gateway returns block that fails validation + cid = CID.parse('bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq') + expectedBlock = Uint8Array.from([0, 1, 2, 0]) + routing = stubInterface() - blocks = [] - for (let i = 0; i < 10; i++) { - blocks.push(await createBlock(raw.code, Uint8Array.from([0, 1, 2, i]))) + badGatewayPeer = { + id: await createEd25519PeerId(), + multiaddrs: [ + uriToMultiaddr(process.env.BAD_TRUSTLESS_GATEWAY ?? '') + ] + } + goodGatewayPeer = { + id: await createEd25519PeerId(), + multiaddrs: [ + uriToMultiaddr(process.env.TRUSTLESS_GATEWAY ?? '') + ] } - gateways = [ - stubConstructor(TrustlessGateway, 'http://localhost:8080', defaultLogger()), - stubConstructor(TrustlessGateway, 'http://localhost:8081', defaultLogger()), - stubConstructor(TrustlessGateway, 'http://localhost:8082', defaultLogger()), - stubConstructor(TrustlessGateway, 'http://localhost:8083', defaultLogger()) - ] gatewayBlockBroker = new TrustlessGatewayBlockBroker({ routing, logger: defaultLogger() + }, { + allowInsecure: true, + allowLocal: true }) - // must copy the array because the broker calls .sort which mutates in-place - ;(gatewayBlockBroker as any).gateways = [...gateways] }) it('tries all gateways before failing', async () => { - // stub all gateway responses to fail - for (const gateway of gateways) { - gateway.getRawBlock.rejects(new Error('failed')) - } - - await expect(gatewayBlockBroker.retrieve?.(blocks[0].cid)) - .to.eventually.be.rejected() - .with.property('errors') - .with.lengthOf(gateways.length) - - for (const gateway of gateways) { - expect(gateway.getRawBlock.calledWith(blocks[0].cid)).to.be.true() - } - }) - - it('prioritizes gateways based on reliability', async () => { - const callOrder: number[] = [] - - // stub all gateway responses to fail, and set reliabilities to known values. - stubGateways({ - default: (gateway, i) => { - gateway.getRawBlock.withArgs(blocks[1].cid, Sinon.match.any).callsFake(async () => { - callOrder.push(i) - throw new Error('failed') - }) - gateway.reliability.returns(i) // known reliability of 0, 1, 2, 3 - } + routing.findProviders.callsFake(async function * () { + yield badGatewayPeer + yield badGatewayPeer }) - await expect(gatewayBlockBroker.retrieve?.(blocks[1].cid)).to.eventually.be.rejected() + // see .aegir.js - bad gateway returns 500 + const cid = CID.parse('bafkreihrn5vx6hgqqltsfbhyr4fqlfu47icddprfubkqrztm6yft5iwpmm') - // all gateways were called - expect(gateways[0].getRawBlock.calledWith(blocks[1].cid)).to.be.true() - expect(gateways[1].getRawBlock.calledWith(blocks[1].cid)).to.be.true() - expect(gateways[2].getRawBlock.calledWith(blocks[1].cid)).to.be.true() - expect(gateways[3].getRawBlock.calledWith(blocks[1].cid)).to.be.true() - // and in the correct order. - expect(callOrder).to.have.ordered.members([3, 2, 1, 0]) + await expect(gatewayBlockBroker.retrieve?.(cid)) + .to.eventually.be.rejected() + .with.property('errors') + .with.lengthOf(2) }) it('tries other gateways if it receives invalid blocks', async () => { - const { cid: cid1, block: block1 } = blocks[0] - const { block: block2 } = blocks[1] - stubGateways({ - // return valid block for only one gateway - 0: (gateway) => { - gateway.getRawBlock.withArgs(cid1, Sinon.match.any).resolves(block1) - gateway.reliability.returns(0) // make sure it's called last - }, - // return invalid blocks for all other gateways - default: (gateway) => { // default stub function - gateway.getRawBlock.withArgs(cid1, Sinon.match.any).resolves(block2) // invalid block for the CID - gateway.reliability.returns(1) // make sure other gateways are called first - } + routing.findProviders.callsFake(async function * () { + yield badGatewayPeer + yield goodGatewayPeer }) - const block = await gatewayBlockBroker.retrieve?.(cid1, { + const block = await gatewayBlockBroker.retrieve?.(cid, { validateFn: async (block) => { - if (block !== block1) { - throw new Error('invalid block') + const hash = await sha256.digest(block) + + if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) { + throw new Error('Bad hash') } } }) - expect(block).to.equal(block1) - // expect that all gateways are called, because everyone returned invalid blocks except the last one - for (const gateway of gateways) { - expect(gateway.getRawBlock.calledWith(cid1, Sinon.match.any)).to.be.true() - } + expect(block).to.equalBytes(expectedBlock) }) it('does not call other gateways if the first gateway returns a valid block', async () => { - const { cid: cid1, block: block1 } = blocks[0] - const { block: block2 } = blocks[1] - - stubGateways({ - // return valid block for only one gateway - 3: (gateway) => { - gateway.getRawBlock.withArgs(cid1, Sinon.match.any).resolves(block1) - gateway.reliability.returns(1) // make sure it's called first - }, - // return invalid blocks for all other gateways - default: (gateway) => { // default stub function - gateway.getRawBlock.withArgs(cid1, Sinon.match.any).resolves(block2) // invalid block for the CID - gateway.reliability.returns(0) // make sure other gateways are called last - } + routing.findProviders.callsFake(async function * () { + yield goodGatewayPeer + yield badGatewayPeer }) - const block = await gatewayBlockBroker.retrieve?.(cid1, { + + const cid = CID.parse('bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq') + const expectedBlock = Uint8Array.from([0, 1, 2, 0]) + + const block = await gatewayBlockBroker.retrieve?.(cid, { validateFn: async (block) => { - if (block !== block1) { - throw new Error('invalid block') + const hash = await sha256.digest(block) + + if (!uint8ArrayEquals(hash.digest, cid.multihash.digest)) { + throw new Error('Bad hash') } } }) - expect(block).to.equal(block1) - expect(gateways[3].getRawBlock.calledWith(cid1, Sinon.match.any)).to.be.true() - // expect that other gateways are not called, because the first gateway returned a valid block - expect(gateways[0].getRawBlock.calledWith(cid1, Sinon.match.any)).to.be.false() - expect(gateways[1].getRawBlock.calledWith(cid1, Sinon.match.any)).to.be.false() - expect(gateways[2].getRawBlock.calledWith(cid1, Sinon.match.any)).to.be.false() + + expect(block).to.equalBytes(expectedBlock) }) it('creates a session', async () => { @@ -188,7 +139,7 @@ describe('trustless-gateway-block-broker', () => { expect(sessionBlockstore).to.be.ok() - await expect(sessionBlockstore?.retrieve?.(blocks[0].cid)).to.eventually.deep.equal(blocks[0].block) + await expect(sessionBlockstore?.retrieve?.(cid)).to.eventually.deep.equal(expectedBlock) }) it('does not trigger new network requests if the same cid request is in-flight', async function () { diff --git a/packages/helia/src/index.ts b/packages/helia/src/index.ts index 4e5050453..5ec50bfc0 100644 --- a/packages/helia/src/index.ts +++ b/packages/helia/src/index.ts @@ -20,7 +20,7 @@ */ import { bitswap, trustlessGateway } from '@helia/block-brokers' -import { libp2pRouting } from '@helia/routers' +import { httpGatewayRouting, libp2pRouting } from '@helia/routers' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { HeliaP2P } from './helia-p2p.js' @@ -121,7 +121,8 @@ export async function createHelia (init: Partial = {}): Promise = {}): Promi trustlessGateway() ], routers: init.routers ?? [ - delegatedHTTPRouting('https://delegated-ipfs.dev') + delegatedHTTPRouting('https://delegated-ipfs.dev'), + httpGatewayRouting() ] }) diff --git a/packages/routers/src/http-gateway-routing.ts b/packages/routers/src/http-gateway-routing.ts index 081d912d6..88692a328 100644 --- a/packages/routers/src/http-gateway-routing.ts +++ b/packages/routers/src/http-gateway-routing.ts @@ -8,6 +8,14 @@ import type { Provider, Routing, RoutingOptions } from '@helia/interface' import type { PeerId, PeerInfo } from '@libp2p/interface' import type { MultihashDigest, Version } from 'multiformats' +export const DEFAULT_TRUSTLESS_GATEWAYS = [ + // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/ + 'https://trustless-gateway.link', + + // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/ + 'https://4everland.io' +] + export interface HTTPGatwayRouterInit { gateways?: Array } @@ -74,11 +82,11 @@ class HTTPGatwayRouter implements Partial { private readonly gateways: PeerInfo[] constructor (init: HTTPGatwayRouterInit = {}) { - this.gateways = (init.gateways ?? []).map(url => toPeerInfo(url)) + this.gateways = (init.gateways ?? DEFAULT_TRUSTLESS_GATEWAYS).map(url => toPeerInfo(url)) } async * findProviders (cid: CID, options?: RoutingOptions | undefined): AsyncIterable { - yield * this.gateways.map(info => ({ + yield * this.gateways.toSorted(() => Math.random() > 0.5 ? 1 : -1).map(info => ({ ...info, protocols: ['transport-ipfs-gateway-http'] }))