feat: support custom dnslink resolvers (#224)

* feat: custom dnslink resolvers
* docs: RAINBOW_DNSLINK_RESOLVERS

including explicit defaults

---------

Co-authored-by: Marcin Rataj <[email protected]>

CHANGELOG.md

@@ -15,6 +15,8 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- Add support for custom DNSLink resolvers (e.g. to support TLDs like `.eth`, `.crypto`). It is possible to set custom DoH resolvers by setting `RAINBOW_DNSLINK_RESOLVERS` with the same convention as Kubo's [`DNS.Resolvers`](https://github.com/ipfs/kubo/blob/master/docs/config.md#dnsresolvers) ) [#224](https://github.com/ipfs/rainbow/pull/224)
+
 ### Changed
 
 ### Fixed

docs/environment-variables.md

@@ -15,6 +15,7 @@
   - [`RAINBOW_SEED_INDEX`](#rainbow_seed_index)
   - [`RAINBOW_DHT_ROUTING`](#rainbow_dht_routing)
   - [`RAINBOW_HTTP_ROUTERS`](#rainbow_http_routers)
+  - [`RAINBOW_DNSLINK_RESOLVERS`](#rainbow_dnslink_resolvers)
 - [Experiments](#experiments)
   - [`RAINBOW_SEED_PEERING`](#rainbow_seed_peering)
   - [`RAINBOW_SEED_PEERING_MAX_INDEX`](#rainbow_seed_peering_max_index)
@@ -32,6 +33,7 @@
   - [`IPFS_NS_MAP`](#ipfs_ns_map)
 - [Tracing](#tracing)
   - [`RAINBOW_TRACING_AUTH`](#rainbow_tracing_auth)
+  - [`RAINBOW_SAMPLING_FRACTION`](#rainbow_sampling_fraction)
 
 ## Configuration
 
@@ -148,6 +150,14 @@ HTTP servers with /routing/v1 endpoints to use for delegated routing (comma-sepa
 
 Default: `https://cid.contact`
 
+### `RAINBOW_DNSLINK_RESOLVERS`
+
+DNS-over-HTTPS servers to use for resolving DNSLink on specified TLDs (comma-separated map: `TLD:URL,TLD2:URL2`).
+
+It is possible to override OS resolver by passing root:  `. : catch-URL`.
+
+Default: `eth. : https://resolver.cloudflare-eth.com/dns-query, crypto. : https://resolver.cloudflare-eth.com/dns-query`
+
 ## Experiments
 
 ### `RAINBOW_SEED_PEERING`

go.mod

@@ -33,6 +33,7 @@ require (
 	github.com/mitchellh/go-server-timing v1.0.1
 	github.com/mr-tron/base58 v1.2.0
 	github.com/multiformats/go-multiaddr v0.14.0
+	github.com/multiformats/go-multiaddr-dns v0.4.1
 	github.com/multiformats/go-multicodec v0.9.0
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/prometheus/client_golang v1.20.5
@@ -149,7 +150,6 @@ require (
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
-	github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.2.0 // indirect
 	github.com/multiformats/go-multihash v0.2.3 // indirect

main.go

@@ -19,9 +19,11 @@ import (
 	"time"
 
 	sddaemon "github.com/coreos/go-systemd/v22/daemon"
+	"github.com/ipfs/boxo/gateway"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	peer "github.com/libp2p/go-libp2p/core/peer"
+	madns "github.com/multiformats/go-multiaddr-dns"
 	"github.com/urfave/cli/v2"
 	"go.opentelemetry.io/contrib/propagators/autoprop"
 	"go.opentelemetry.io/otel"
@@ -399,6 +401,12 @@ Generate an identity seed and launch a gateway:
 			EnvVars: []string{"ROUTING_MAX_TIMEOUT"},
 			Usage:   "Maximum time for routing to find the maximum number of providers",
 		},
+		&cli.StringSliceFlag{
+			Name:    "dnslink-resolvers",
+			Value:   cli.NewStringSlice(extraDNSLinkResolvers...),
+			EnvVars: []string{"RAINBOW_DNSLINK_RESOLVERS"},
+			Usage:   "The DNSLink resolvers to use (comma-separated tuples that each look like `eth. : https://dns.eth.limo/dns-query`)",
+		},
 	}
 
 	app.Commands = []*cli.Command{
@@ -513,6 +521,12 @@ share the same seed as long as the indexes are different.
 			}
 		}
 
+		customDNSResolvers := cctx.StringSlice("dnslink-resolvers")
+		dns, err := parseCustomDNSLinkResolvers(customDNSResolvers)
+		if err != nil {
+			return err
+		}
+
 		cfg := Config{
 			DataDir:                    ddir,
 			BlockstoreType:             cctx.String("blockstore"),
@@ -546,6 +560,7 @@ share the same seed as long as the indexes are different.
 			GCThreshold:                cctx.Float64("gc-threshold"),
 			ListenAddrs:                cctx.StringSlice("libp2p-listen-addrs"),
 			TracingAuthToken:           cctx.String("tracing-auth"),
+			DNSLinkResolver:            dns,
 
 			// Pebble config
 			BytesPerSync:                cctx.Int("pebble-bytes-per-sync"),
@@ -639,6 +654,7 @@ share the same seed as long as the indexes are different.
 		printIfListConfigured("  RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS = ", cfg.SubdomainGatewayDomains)
 		printIfListConfigured("  RAINBOW_TRUSTLESS_GATEWAY_DOMAINS = ", cfg.TrustlessGatewayDomains)
 		printIfListConfigured("  RAINBOW_HTTP_ROUTERS              = ", cfg.RoutingV1Endpoints)
+		printIfListConfigured("  RAINBOW_DNSLINK_RESOLVERS         = ", customDNSResolvers)
 		printIfListConfigured("  RAINBOW_REMOTE_BACKENDS           = ", cfg.RemoteBackends)
 
 		fmt.Printf("\n")
@@ -766,3 +782,21 @@ func replaceRainbowSeedWithPeer(addr string, seed string) (string, error) {
 
 	return strings.Replace(addr, match[0], "/p2p/"+pid.String(), 1), nil
 }
+
+func parseCustomDNSLinkResolvers(customDNSResolvers []string) (madns.BasicResolver, error) {
+	customDNSResolverMap := make(map[string]string)
+	for _, s := range customDNSResolvers {
+		split := strings.SplitN(s, ":", 2)
+		if len(split) != 2 {
+			return nil, fmt.Errorf("invalid DNS resolver: %s", s)
+		}
+		domain := strings.TrimSpace(split[0])
+		resolverUrl := strings.TrimSpace(split[1])
+		customDNSResolverMap[domain] = resolverUrl
+	}
+	dns, err := gateway.NewDNSResolver(customDNSResolverMap)
+	if err != nil {
+		return nil, err
+	}
+	return dns, nil
+}

main_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	chunker "github.com/ipfs/boxo/chunker"
+	"github.com/ipfs/boxo/gateway"
 	"github.com/ipfs/boxo/ipld/merkledag"
 	"github.com/ipfs/boxo/ipld/unixfs/importer/balanced"
 	uih "github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
@@ -63,6 +64,12 @@ func mustTestNodeWithKey(t *testing.T, cfg Config, sk ic.PrivKey) *Node {
 		_ = cdns.Close()
 	})
 
+	if cfg.DNSLinkResolver == nil {
+		dnslinkResovler, err := gateway.NewDNSResolver(nil)
+		require.NoError(t, err)
+		cfg.DNSLinkResolver = dnslinkResovler
+	}
+
 	nd, err := SetupWithLibp2p(ctx, cfg, sk, cdns)
 	require.NoError(t, err)
 	return nd

setup.go

@@ -41,6 +41,7 @@ import (
 	"github.com/libp2p/go-libp2p/p2p/net/connmgr"
 	"github.com/libp2p/go-libp2p/p2p/protocol/identify"
 	"github.com/multiformats/go-multiaddr"
+	madns "github.com/multiformats/go-multiaddr-dns"
 )
 
 func init() {
@@ -53,6 +54,11 @@ const cidContactEndpoint = "https://cid.contact"
 
 var httpRoutersFilterProtocols = []string{"unknown", "transport-bitswap"} // IPIP-484
 
+var extraDNSLinkResolvers = []string{
+	"eth. : https://resolver.cloudflare-eth.com/dns-query",
+	"crypto. : https://resolver.cloudflare-eth.com/dns-query",
+}
+
 type DHTRouting string
 
 const (
@@ -112,6 +118,8 @@ type Config struct {
 	IpnsMaxCacheTTL          time.Duration
 	Bitswap                  bool
 
+	DNSLinkResolver madns.BasicResolver
+
 	// BitswapWantHaveReplaceSize tells the bitswap server to replace WantHave
 	// with WantBlock responses when the block size less then or equal to this
 	// value. Set to zero to disable replacement and avoid block size lookup
@@ -194,7 +202,7 @@ func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node,
 		bsrv = nopfsipfs.WrapBlockService(bsrv, blocker)
 	}
 
-	ns, err := setupNamesys(cfg, vs, blocker)
+	ns, err := setupNamesys(cfg, vs, blocker, cfg.DNSLinkResolver)
 	if err != nil {
 		return nil, err
 	}
@@ -357,7 +365,7 @@ func SetupWithLibp2p(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCac
 	n.bsrv = bsrv
 	n.resolver = r
 
-	ns, err := setupNamesys(cfg, vs, blocker)
+	ns, err := setupNamesys(cfg, vs, blocker, cfg.DNSLinkResolver)
 	if err != nil {
 		return nil, err
 	}
@@ -521,12 +529,8 @@ func setupDenylists(cfg Config) ([]*nopfs.HTTPSubscriber, *nopfs.Blocker, error)
 	return denylists, blocker, nil
 }
 
-func setupNamesys(cfg Config, vs routing.ValueStore, blocker *nopfs.Blocker) (namesys.NameSystem, error) {
-	dns, err := gateway.NewDNSResolver(nil)
-	if err != nil {
-		return nil, err
-	}
-	nsOptions := []namesys.Option{namesys.WithDNSResolver(dns)}
+func setupNamesys(cfg Config, vs routing.ValueStore, blocker *nopfs.Blocker, dnslinkResolver madns.BasicResolver) (namesys.NameSystem, error) {
+	nsOptions := []namesys.Option{namesys.WithDNSResolver(dnslinkResolver)}
 	if cfg.IpnsMaxCacheTTL > 0 {
 		nsOptions = append(nsOptions, namesys.WithMaxCacheTTL(cfg.IpnsMaxCacheTTL))
 	}

setup_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ipfs/boxo/gateway"
 	blocks "github.com/ipfs/go-block-format"
 	ci "github.com/libp2p/go-libp2p-testing/ci"
 	ic "github.com/libp2p/go-libp2p/core/crypto"
@@ -173,6 +174,8 @@ func testSeedPeering(t *testing.T, n int, dhtRouting DHTRouting, dhtSharedHost b
 	nodes := make([]*Node, n)
 
 	for i := 0; i < n; i++ {
+		dnslinkResolver, err := gateway.NewDNSResolver(nil)
+		require.NoError(t, err)
 		cfgs[i] = Config{
 			DataDir:             t.TempDir(),
 			BlockstoreType:      "flatfs",
@@ -183,6 +186,7 @@ func testSeedPeering(t *testing.T, n int, dhtRouting DHTRouting, dhtSharedHost b
 			SeedIndex:           i,
 			SeedPeering:         true,
 			SeedPeeringMaxIndex: n,
+			DNSLinkResolver:     dnslinkResolver,
 		}
 
 		nodes[i], err = SetupWithLibp2p(ctx, cfgs[i], keys[i], cdns)