diff --git a/examples/gateway/car/README.md b/examples/gateway/car/README.md index c9f972403..2fea3fa66 100644 --- a/examples/gateway/car/README.md +++ b/examples/gateway/car/README.md @@ -1,4 +1,4 @@ -# Gateway backed by a CAR File +# HTTP Gateway backed by a CAR File This is an example that shows how to build a Gateway backed by the contents of a CAR file. A [CAR file](https://ipld.io/specs/transport/car/) is a Content @@ -7,7 +7,7 @@ Addressable aRchive that contains blocks. ## Build ```bash -> go build -o gateway +> go build -o car-gateway ``` ## Usage @@ -23,10 +23,19 @@ Then, you can start the gateway with: ``` -./gateway -c data.car -p 8040 +./car-gateway -c data.car -p 8040 ``` -Now you can access the gateway in [127.0.0.1:8040](http://127.0.0.1:8040). It will -behave like a regular IPFS Gateway, except for the fact that all contents are provided +### Subdomain gateway + +Now you can access the gateway in [localhost:8040](http://localhost:8040). It will +behave like a regular [Subdomain IPFS Gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway), +except for the fact that all contents are provided from the CAR file. Therefore, things such as IPNS resolution and fetching contents from nodes in the IPFS network won't work. + +### Path gateway + +If you don't need Origin isolation and only care about hosting flat files, +a plain [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at [127.0.0.1:8040](http://127.0.0.1:8040) +may suffice. diff --git a/examples/gateway/car/main.go b/examples/gateway/car/main.go index a0c03fa0b..070dacb94 100644 --- a/examples/gateway/car/main.go +++ b/examples/gateway/car/main.go @@ -12,12 +12,13 @@ import ( "github.com/ipfs/go-cid" offline "github.com/ipfs/go-ipfs-exchange-offline" "github.com/ipfs/go-libipfs/examples/gateway/common" + "github.com/ipfs/go-libipfs/gateway" carblockstore "github.com/ipld/go-car/v2/blockstore" ) func main() { carFilePtr := flag.String("c", "", "path to CAR file to back this gateway from") - portPtr := flag.Int("p", 8080, "port to run this gateway from") + port := flag.Int("p", 8040, "port to run this gateway from") flag.Parse() blockService, roots, f, err := newBlockServiceFromCAR(*carFilePtr) @@ -26,20 +27,39 @@ func main() { } defer f.Close() - gateway, err := common.NewBlocksGateway(blockService, nil) + gwAPI, err := common.NewBlocksGateway(blockService, nil) if err != nil { log.Fatal(err) } + handler := common.NewBlocksHandler(gwAPI, *port) - handler := common.NewBlocksHandler(gateway, *portPtr) + // Initialize the public gateways that we will want to have available through + // Host header rewritting. This step is optional and only required if you're + // running multiple public gateways and want different settings and support + // for DNSLink and Subdomain Gateways. + noDNSLink := true // If you set DNSLink to point at the CID from CAR, you can load it! + publicGateways := map[string]*gateway.Specification{ + // Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net + "example.net": { + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: noDNSLink, + UseSubdomains: true, + }, + // Support local requests + "localhost": { + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: noDNSLink, + UseSubdomains: true, + }, + } + handler = gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink) - address := "127.0.0.1:" + strconv.Itoa(*portPtr) - log.Printf("Listening on http://%s", address) + log.Printf("Listening on http://localhost:%d", *port) for _, cid := range roots { - log.Printf("Hosting CAR root at http://%s/ipfs/%s", address, cid.String()) + log.Printf("Hosting CAR root at http://localhost:%d/ipfs/%s", *port, cid.String()) } - if err := http.ListenAndServe(address, handler); err != nil { + if err := http.ListenAndServe(":"+strconv.Itoa(*port), handler); err != nil { log.Fatal(err) } } diff --git a/examples/gateway/common/blocks.go b/examples/gateway/common/blocks.go index 40288136c..a34acae01 100644 --- a/examples/gateway/common/blocks.go +++ b/examples/gateway/common/blocks.go @@ -2,6 +2,7 @@ package common import ( "context" + "errors" "fmt" "net/http" gopath "path" @@ -24,6 +25,7 @@ import ( uio "github.com/ipfs/go-unixfs/io" "github.com/ipfs/go-unixfsnode" iface "github.com/ipfs/interface-go-ipfs-core" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" ifacepath "github.com/ipfs/interface-go-ipfs-core/path" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" @@ -143,6 +145,18 @@ func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, return nil, routing.ErrNotSupported } +func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { + if api.namesys != nil { + p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ifacepath.New(p.String()), err + } + + return nil, errors.New("not implemented") +} + func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool { rp, err := api.ResolvePath(ctx, p) if err != nil { diff --git a/examples/gateway/proxy/README.md b/examples/gateway/proxy/README.md index 0c54eb60e..4164aad1e 100644 --- a/examples/gateway/proxy/README.md +++ b/examples/gateway/proxy/README.md @@ -28,9 +28,31 @@ types. Once you have it, run the proxy gateway with its address as the host para ``` -./verifying-proxy -h https://ipfs.io -p 8040 +./verifying-proxy -g https://ipfs.io -p 8040 ``` -Now you can access the gateway in [127.0.0.1:8040](http://127.0.0.1:8040). It will -behave like a regular IPFS Gateway, except for the fact that it runs no libp2p, and has no local blockstore. +### Subdomain gateway + +Now you can access the gateway in [localhost:8040](http://localhost:8040). It will +behave like a regular [Subdomain IPFS Gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway), +except for the fact that it runs no libp2p, and has no local blockstore. All contents are provided by a remote gateway and fetched as RAW Blocks and Records, and verified locally. + +### Path gateway + +If you don't need Origin isolation and only care about hosting flat files, +a plain [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at [127.0.0.1:8040](http://127.0.0.1:8040) +may suffice. + +### DNSLink gateway + +Gateway supports hosting of [DNSLink](https://dnslink.dev/) websites. All you need is to pass `Host` header with FQDN that has DNSLink set up: + +```console +$ curl -sH 'Host: en.wikipedia-on-ipfs.org' 'http://127.0.0.1:8080/wiki/' | head -3 + + + Wikipedia, the free encyclopedia +``` + +Put it behind a reverse proxy terminating TLS (like Nginx) and voila! diff --git a/examples/gateway/proxy/main.go b/examples/gateway/proxy/main.go index f4d8c0b7c..6f9282c19 100644 --- a/examples/gateway/proxy/main.go +++ b/examples/gateway/proxy/main.go @@ -9,11 +9,12 @@ import ( "github.com/ipfs/go-blockservice" offline "github.com/ipfs/go-ipfs-exchange-offline" "github.com/ipfs/go-libipfs/examples/gateway/common" + "github.com/ipfs/go-libipfs/gateway" ) func main() { gatewayUrlPtr := flag.String("g", "", "gateway to proxy to") - portPtr := flag.Int("p", 8080, "port to run this gateway from") + port := flag.Int("p", 8040, "port to run this gateway from") flag.Parse() // Sets up the block store, which will proxy the block requests to the given gateway. @@ -24,16 +25,35 @@ func main() { routing := newProxyRouting(*gatewayUrlPtr, nil) // Creates the gateway with the block service and the routing. - gateway, err := common.NewBlocksGateway(blockService, routing) + gwAPI, err := common.NewBlocksGateway(blockService, routing) if err != nil { log.Fatal(err) } + handler := common.NewBlocksHandler(gwAPI, *port) + + // Initialize the public gateways that we will want to have available through + // Host header rewritting. This step is optional and only required if you're + // running multiple public gateways and want different settings and support + // for DNSLink and Subdomain Gateways. + noDNSLink := false + publicGateways := map[string]*gateway.Specification{ + // Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net + "example.net": { + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: noDNSLink, + UseSubdomains: true, + }, + // Support local requests + "localhost": { + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: noDNSLink, + UseSubdomains: true, + }, + } + handler = gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink) - handler := common.NewBlocksHandler(gateway, *portPtr) - address := "127.0.0.1:" + strconv.Itoa(*portPtr) - log.Printf("Listening on http://%s", address) - - if err := http.ListenAndServe(address, handler); err != nil { + log.Printf("Listening on http://localhost:%d", *port) + if err := http.ListenAndServe(":"+strconv.Itoa(*port), handler); err != nil { log.Fatal(err) } } diff --git a/gateway/gateway.go b/gateway/gateway.go index f50d80332..718bd3021 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -32,6 +32,12 @@ type API interface { // from the routing system. GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) + // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN. + // Unlike ResolvePath, it does not perform recursive resolution. It only + // checks for the existence of a DNSLink TXT record with path starting with + // /ipfs/ or /ipns/ and returns the path as-is. + GetDNSLinkRecord(context.Context, string) (path.Path, error) + // IsCached returns whether or not the path exists locally. IsCached(context.Context, path.Path) bool diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go new file mode 100644 index 000000000..f878cb631 --- /dev/null +++ b/gateway/gateway_test.go @@ -0,0 +1,103 @@ +package gateway + +import ( + "context" + "errors" + "strings" + + cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-namesys" + path "github.com/ipfs/go-path" + iface "github.com/ipfs/interface-go-ipfs-core" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" + ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/libp2p/go-libp2p/core/crypto" +) + +type mockNamesys map[string]path.Path + +func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...nsopts.ResolveOpt) (value path.Path, err error) { + cfg := nsopts.DefaultResolveOpts() + for _, o := range opts { + o(&cfg) + } + depth := cfg.Depth + if depth == nsopts.UnlimitedDepth { + // max uint + depth = ^uint(0) + } + for strings.HasPrefix(name, "/ipns/") { + if depth == 0 { + return value, namesys.ErrResolveRecursion + } + depth-- + + var ok bool + value, ok = m[name] + if !ok { + return "", namesys.ErrResolveFailed + } + name = value.String() + } + return value, nil +} + +func (m mockNamesys) ResolveAsync(ctx context.Context, name string, opts ...nsopts.ResolveOpt) <-chan namesys.Result { + out := make(chan namesys.Result, 1) + v, err := m.Resolve(ctx, name, opts...) + out <- namesys.Result{Path: v, Err: err} + close(out) + return out +} + +func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...nsopts.PublishOption) error { + return errors.New("not implemented for mockNamesys") +} + +func (m mockNamesys) GetResolver(subs string) (namesys.Resolver, bool) { + return nil, false +} + +type mockApi struct { + ns mockNamesys +} + +func newMockApi() *mockApi { + return &mockApi{ + ns: mockNamesys{}, + } +} + +func (m *mockApi) GetUnixFsNode(context.Context, ipath.Resolved) (files.Node, error) { + return nil, errors.New("not implemented") +} + +func (m *mockApi) LsUnixFsDir(context.Context, ipath.Resolved) (<-chan iface.DirEntry, error) { + return nil, errors.New("not implemented") +} + +func (m *mockApi) GetBlock(context.Context, cid.Cid) (blocks.Block, error) { + return nil, errors.New("not implemented") +} + +func (m *mockApi) GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) { + return nil, errors.New("not implemented") +} + +func (m *mockApi) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { + p, err := m.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ipath.New(p.String()), err +} + +func (m *mockApi) IsCached(context.Context, ipath.Path) bool { + return false +} + +func (m *mockApi) ResolvePath(context.Context, ipath.Path) (ipath.Resolved, error) { + return nil, errors.New("not implemented") +} diff --git a/gateway/hostname.go b/gateway/hostname.go new file mode 100644 index 000000000..563804e12 --- /dev/null +++ b/gateway/hostname.go @@ -0,0 +1,592 @@ +package gateway + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "regexp" + "strings" + + cid "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/peer" + dns "github.com/miekg/dns" + + mbase "github.com/multiformats/go-multibase" +) + +// Specification is the specification of an IPFS Public Gateway. +type Specification struct { + // Paths is explicit list of path prefixes that should be handled by + // this gateway. Example: `["/ipfs", "/ipns"]` + // Useful if you only want to support immutable `/ipfs`. + Paths []string + + // UseSubdomains indicates whether or not this gateway uses subdomains + // for IPFS resources instead of paths. That is: http://CID.ipfs.GATEWAY/... + // + // If this flag is set, any /ipns/$id and/or /ipfs/$id paths in Paths + // will be permanently redirected to http://$id.[ipns|ipfs].$gateway/. + // + // We do not support using both paths and subdomains for a single domain + // for security reasons (Origin isolation). + UseSubdomains bool + + // NoDNSLink configures this gateway to _not_ resolve DNSLink for the + // specific FQDN provided in `Host` HTTP header. Useful when you want to + // explicitly allow or refuse hosting a single hostname. To refuse all + // DNSLinks in `Host` processing, pass noDNSLink to `WithHostname` instead. + // This flag overrides the global one. + NoDNSLink bool + + // InlineDNSLink configures this gateway to always inline DNSLink names + // (FQDN) into a single DNS label in order to interop with wildcard TLS certs + // and Origin per CID isolation provided by rules like https://publicsuffix.org + // This should be set to true if you use HTTPS. + InlineDNSLink bool +} + +// WithHostname is a middleware that can wrap an http.Handler in order to parse the +// Host header and translating it to the content path. This is useful for Subdomain +// and DNSLink gateways. +// +// publicGateways configures the behavior of known public gateways. Each key is a +// fully qualified domain name (FQDN). +// +// noDNSLink configures the gateway to _not_ perform DNS TXT record lookups in +// response to requests with values in `Host` HTTP header. This flag can be overridden +// per FQDN in publicGateways. +func WithHostname(next http.Handler, api API, publicGateways map[string]*Specification, noDNSLink bool) http.HandlerFunc { + gateways := prepareHostnameGateways(publicGateways) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Unfortunately, many (well, ipfs.io) gateways use + // DNSLink so if we blindly rewrite with DNSLink, we'll + // break /ipfs links. + // + // We fix this by maintaining a list of known gateways + // and the paths that they serve "gateway" content on. + // That way, we can use DNSLink for everything else. + + // Support X-Forwarded-Host if added by a reverse proxy + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host + host := r.Host + if xHost := r.Header.Get("X-Forwarded-Host"); xHost != "" { + host = xHost + } + + // HTTP Host & Path check: is this one of our "known gateways"? + if gw, ok := gateways.isKnownHostname(host); ok { + // This is a known gateway but request is not using + // the subdomain feature. + + // Does this gateway _handle_ this path? + if hasPrefix(r.URL.Path, gw.Paths...) { + // It does. + + // Should this gateway use subdomains instead of paths? + if gw.UseSubdomains { + // Yes, redirect if applicable + // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link + useInlinedDNSLink := gw.InlineDNSLink + newURL, err := toSubdomainURL(host, r.URL.Path, r, useInlinedDNSLink, api) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Set "Location" header with redirect destination. + // It is ignored by curl in default mode, but will + // be respected by user agents that follow + // redirects by default, namely web browsers + w.Header().Set("Location", newURL) + + // Note: we continue regular gateway processing: + // HTTP Status Code http.StatusMovedPermanently + // will be set later, in statusResponseWriter + } + } + + // Not a subdomain resource, continue with path processing + // Example: 127.0.0.1:8080/ipfs/{CID}, ipfs.io/ipfs/{CID} etc + next.ServeHTTP(w, r) + return + } + // Not a whitelisted path + + // Try DNSLink, if it was not explicitly disabled for the hostname + if !gw.NoDNSLink && hasDNSLinkRecord(r.Context(), api, host) { + // rewrite path and handle as DNSLink + r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path + next.ServeHTTP(w, withHostnameContext(r, host)) + return + } + + // If not, resource does not exist on the hostname, return 404 + http.NotFound(w, r) + return + } + + // HTTP Host check: is this one of our subdomain-based "known gateways"? + // IPFS details extracted from the host: {rootID}.{ns}.{gwHostname} + // /ipfs/ example: {cid}.ipfs.localhost:8080, {cid}.ipfs.dweb.link + // /ipns/ example: {libp2p-key}.ipns.localhost:8080, {inlined-dnslink-fqdn}.ipns.dweb.link + if gw, gwHostname, ns, rootID, ok := gateways.knownSubdomainDetails(host); ok { + // Looks like we're using a known gateway in subdomain mode. + + // Assemble original path prefix. + pathPrefix := "/" + ns + "/" + rootID + + // Retrieve whether or not we should inline DNSLink. + useInlinedDNSLink := gw.InlineDNSLink + + // Does this gateway _handle_ subdomains AND this path? + if !(gw.UseSubdomains && hasPrefix(pathPrefix, gw.Paths...)) { + // If not, resource does not exist, return 404 + http.NotFound(w, r) + return + } + + // Check if rootID is a valid CID + if rootCID, err := cid.Decode(rootID); err == nil { + // Do we need to redirect root CID to a canonical DNS representation? + dnsCID, err := toDNSLabel(rootID, rootCID) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if !strings.HasPrefix(r.Host, dnsCID) { + dnsPrefix := "/" + ns + "/" + dnsCID + newURL, err := toSubdomainURL(gwHostname, dnsPrefix+r.URL.Path, r, useInlinedDNSLink, api) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to deterministic CID to ensure CID + // always gets the same Origin on the web + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + + // Do we need to fix multicodec in PeerID represented as CIDv1? + if isPeerIDNamespace(ns) { + if rootCID.Type() != cid.Libp2pKey { + newURL, err := toSubdomainURL(gwHostname, pathPrefix+r.URL.Path, r, useInlinedDNSLink, api) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to CID fixed inside of toSubdomainURL() + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + } + } else { // rootID is not a CID.. + // Check if rootID is a single DNS label with an inlined + // DNSLink FQDN a single DNS label. We support this so + // loading DNSLink names over TLS "just works" on public + // HTTP gateways. + // + // Rationale for doing this can be found under "Option C" + // at: https://github.com/ipfs/in-web-browsers/issues/169 + // + // TLDR is: + // https://dweb.link/ipns/my.v-long.example.com + // can be loaded from a subdomain gateway with a wildcard + // TLS cert if represented as a single DNS label: + // https://my-v--long-example-com.ipns.dweb.link + if ns == "ipns" && !strings.Contains(rootID, ".") { + // if there is no TXT recordfor rootID + if !hasDNSLinkRecord(r.Context(), api, rootID) { + // my-v--long-example-com → my.v-long.example.com + dnslinkFQDN := toDNSLinkFQDN(rootID) + if hasDNSLinkRecord(r.Context(), api, dnslinkFQDN) { + // update path prefix to use real FQDN with DNSLink + pathPrefix = "/ipns/" + dnslinkFQDN + } + } + } + } + + // Rewrite the path to not use subdomains + r.URL.Path = pathPrefix + r.URL.Path + + // Serve path request + next.ServeHTTP(w, withHostnameContext(r, gwHostname)) + return + } + + // We don't have a known gateway. Fallback on DNSLink lookup + + // Wildcard HTTP Host check: + // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? + // 2. does Host header include a fully qualified domain name (FQDN)? + // 3. does DNSLink record exist in DNS? + if !noDNSLink && hasDNSLinkRecord(r.Context(), api, host) { + // rewrite path and handle as DNSLink + r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path + ctx := context.WithValue(r.Context(), DNSLinkHostnameKey, host) + next.ServeHTTP(w, withHostnameContext(r.WithContext(ctx), host)) + return + } + + // else, treat it as an old school gateway, I guess. + next.ServeHTTP(w, r) + + }) +} + +// Extends request context to include hostname of a canonical gateway root +// (subdomain root or dnslink fqdn) +func withHostnameContext(r *http.Request, hostname string) *http.Request { + // This is required for links on directory listing pages to work correctly + // on subdomain and dnslink gateways. While DNSlink could read value from + // Host header, subdomain gateways have more comples rules (knownSubdomainDetails) + // More: https://github.com/ipfs/dir-index-html/issues/42 + // nolint: staticcheck // non-backward compatible change + ctx := context.WithValue(r.Context(), GatewayHostnameKey, hostname) + return r.WithContext(ctx) +} + +// isDomainNameAndNotPeerID returns bool if string looks like a valid DNS name AND is not a PeerID +func isDomainNameAndNotPeerID(hostname string) bool { + if len(hostname) == 0 { + return false + } + if _, err := peer.Decode(hostname); err == nil { + return false + } + _, ok := dns.IsDomainName(hostname) + return ok +} + +// hasDNSLinkRecord returns if a DNS TXT record exists for the provided host. +func hasDNSLinkRecord(ctx context.Context, api API, host string) bool { + dnslinkName := stripPort(host) + + if !isDomainNameAndNotPeerID(dnslinkName) { + return false + } + + _, err := api.GetDNSLinkRecord(ctx, dnslinkName) + return err == nil +} + +func isSubdomainNamespace(ns string) bool { + switch ns { + case "ipfs", "ipns", "p2p", "ipld": + // Note: 'p2p' and 'ipld' is only kept here for compatibility with Kubo. + return true + default: + return false + } +} + +func isPeerIDNamespace(ns string) bool { + switch ns { + case "ipns", "p2p": + // Note: 'p2p' and 'ipld' is only kept here for compatibility with Kubo. + return true + default: + return false + } +} + +// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7) +const dnsLabelMaxLength int = 63 + +// Converts a CID to DNS-safe representation that fits in 63 characters +func toDNSLabel(rootID string, rootCID cid.Cid) (dnsCID string, err error) { + // Return as-is if things fit + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Convert to Base36 and see if that helped + rootID, err = cid.NewCidV1(rootCID.Type(), rootCID.Hash()).StringOfBase(mbase.Base36) + if err != nil { + return "", err + } + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Can't win with DNS at this point, return error + return "", fmt.Errorf("CID incompatible with DNS label length limit of 63: %s", rootID) +} + +// Returns true if HTTP request involves TLS certificate. +// See https://github.com/ipfs/in-web-browsers/issues/169 to understand how it +// impacts DNSLink websites on public gateways. +func isHTTPSRequest(r *http.Request) bool { + // X-Forwarded-Proto if added by a reverse proxy + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto + xproto := r.Header.Get("X-Forwarded-Proto") + // Is request a native TLS (not used atm, but future-proofing) + // or a proxied HTTPS (eg. go-ipfs behind nginx at a public gw)? + return r.URL.Scheme == "https" || xproto == "https" +} + +// Converts a FQDN to DNS-safe representation that fits in 63 characters: +// my.v-long.example.com → my-v--long-example-com +func toDNSLinkDNSLabel(fqdn string) (dnsLabel string, err error) { + dnsLabel = strings.ReplaceAll(fqdn, "-", "--") + dnsLabel = strings.ReplaceAll(dnsLabel, ".", "-") + if len(dnsLabel) > dnsLabelMaxLength { + return "", fmt.Errorf("DNSLink representation incompatible with DNS label length limit of 63: %s", dnsLabel) + } + return dnsLabel, nil +} + +// Converts a DNS-safe representation of DNSLink FQDN to real FQDN: +// my-v--long-example-com → my.v-long.example.com +func toDNSLinkFQDN(dnsLabel string) (fqdn string) { + fqdn = strings.ReplaceAll(dnsLabel, "--", "@") // @ placeholder is unused in DNS labels + fqdn = strings.ReplaceAll(fqdn, "-", ".") + fqdn = strings.ReplaceAll(fqdn, "@", "-") + return fqdn +} + +// Converts a hostname/path to a subdomain-based URL, if applicable. +func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, api API) (redirURL string, err error) { + var scheme, ns, rootID, rest string + + query := r.URL.RawQuery + parts := strings.SplitN(path, "/", 4) + isHTTPS := isHTTPSRequest(r) + safeRedirectURL := func(in string) (out string, err error) { + safeURI, err := url.ParseRequestURI(in) + if err != nil { + return "", err + } + return safeURI.String(), nil + } + + if isHTTPS { + scheme = "https:" + } else { + scheme = "http:" + } + + switch len(parts) { + case 4: + rest = parts[3] + fallthrough + case 3: + ns = parts[1] + rootID = parts[2] + default: + return "", nil + } + + if !isSubdomainNamespace(ns) { + return "", nil + } + + // add prefix if query is present + if query != "" { + query = "?" + query + } + + // Normalize problematic PeerIDs (eg. ed25519+identity) to CID representation + if isPeerIDNamespace(ns) && !isDomainNameAndNotPeerID(rootID) { + peerID, err := peer.Decode(rootID) + // Note: PeerID CIDv1 with protobuf multicodec will fail, but we fix it + // in the next block + if err == nil { + rootID = peer.ToCid(peerID).String() + } + } + + // If rootID is a CID, ensure it uses DNS-friendly text representation + if rootCID, err := cid.Decode(rootID); err == nil { + multicodec := rootCID.Type() + var base mbase.Encoding = mbase.Base32 + + // Normalizations specific to /ipns/{libp2p-key} + if isPeerIDNamespace(ns) { + // Using Base36 for /ipns/ for consistency + // Context: https://github.com/ipfs/kubo/pull/7441#discussion_r452372828 + base = mbase.Base36 + + // PeerIDs represented as CIDv1 are expected to have libp2p-key + // multicodec (https://github.com/libp2p/specs/pull/209). + // We ease the transition by fixing multicodec on the fly: + // https://github.com/ipfs/kubo/issues/5287#issuecomment-492163929 + if multicodec != cid.Libp2pKey { + multicodec = cid.Libp2pKey + } + } + + // Ensure CID text representation used in subdomain is compatible + // with the way DNS and URIs are implemented in user agents. + // + // 1. Switch to CIDv1 and enable case-insensitive Base encoding + // to avoid issues when user agent force-lowercases the hostname + // before making the request + // (https://github.com/ipfs/in-web-browsers/issues/89) + rootCID = cid.NewCidV1(multicodec, rootCID.Hash()) + rootID, err = rootCID.StringOfBase(base) + if err != nil { + return "", err + } + // 2. Make sure CID fits in a DNS label, adjust encoding if needed + // (https://github.com/ipfs/kubo/issues/7318) + rootID, err = toDNSLabel(rootID, rootCID) + if err != nil { + return "", err + } + } else { // rootID is not a CID + + // Check if rootID is a FQDN with DNSLink and convert it to TLS-safe + // representation that fits in a single DNS label. We support this so + // loading DNSLink names over TLS "just works" on public HTTP gateways + // that pass 'https' in X-Forwarded-Proto to go-ipfs. + // + // Rationale can be found under "Option C" + // at: https://github.com/ipfs/in-web-browsers/issues/169 + // + // TLDR is: + // /ipns/my.v-long.example.com + // can be loaded from a subdomain gateway with a wildcard TLS cert if + // represented as a single DNS label: + // https://my-v--long-example-com.ipns.dweb.link + if (inlineDNSLink || isHTTPS) && ns == "ipns" && strings.Contains(rootID, ".") { + if hasDNSLinkRecord(r.Context(), api, rootID) { + // my.v-long.example.com → my-v--long-example-com + dnsLabel, err := toDNSLinkDNSLabel(rootID) + if err != nil { + return "", err + } + // update path prefix to use real FQDN with DNSLink + rootID = dnsLabel + } + } + } + + return safeRedirectURL(fmt.Sprintf( + "%s//%s.%s.%s/%s%s", + scheme, + rootID, + ns, + hostname, + rest, + query, + )) +} + +func hasPrefix(path string, prefixes ...string) bool { + for _, prefix := range prefixes { + // Assume people are creative with trailing slashes in Gateway config + p := strings.TrimSuffix(prefix, "/") + // Support for both /version and /ipfs/$cid + if p == path || strings.HasPrefix(path, p+"/") { + return true + } + } + return false +} + +func stripPort(hostname string) string { + host, _, err := net.SplitHostPort(hostname) + if err == nil { + return host + } + return hostname +} + +type hostnameGateways struct { + exact map[string]*Specification + wildcard map[*regexp.Regexp]*Specification +} + +// prepareHostnameGateways converts the user given gateways into an internal format +// split between exact and wildcard-based gateway hostnames. +func prepareHostnameGateways(gateways map[string]*Specification) *hostnameGateways { + h := &hostnameGateways{ + exact: map[string]*Specification{}, + wildcard: map[*regexp.Regexp]*Specification{}, + } + + for hostname, gw := range gateways { + if strings.Contains(hostname, "*") { + // from *.domain.tld, construct a regexp that match any direct subdomain + // of .domain.tld. + // + // Regexp will be in the form of ^[^.]+\.domain.tld(?::\d+)?$ + escaped := strings.ReplaceAll(hostname, ".", `\.`) + regexed := strings.ReplaceAll(escaped, "*", "[^.]+") + + re, err := regexp.Compile(fmt.Sprintf(`^%s(?::\d+)?$`, regexed)) + if err != nil { + log.Warn("invalid wildcard gateway hostname \"%s\"", hostname) + } + + h.wildcard[re] = gw + } else { + h.exact[hostname] = gw + } + } + + return h +} + +// isKnownHostname checks the given hostname gateways and returns a matching +// specification with graceful fallback to version without port. +func (gws *hostnameGateways) isKnownHostname(hostname string) (gw *Specification, ok bool) { + // Try hostname (host+optional port - value from Host header as-is) + if gw, ok := gws.exact[hostname]; ok { + return gw, ok + } + // Also test without port + if gw, ok = gws.exact[stripPort(hostname)]; ok { + return gw, ok + } + + // Wildcard support. Test both with and without port. + for re, spec := range gws.wildcard { + if re.MatchString(hostname) { + return spec, true + } + } + + return nil, false +} + +// knownSubdomainDetails parses the Host header and looks for a known gateway matching +// the subdomain host. If found, returns a Specification and the subdomain components +// extracted from Host header: {rootID}.{ns}.{gwHostname}. +// Note: hostname is host + optional port +func (gws *hostnameGateways) knownSubdomainDetails(hostname string) (gw *Specification, gwHostname, ns, rootID string, ok bool) { + labels := strings.Split(hostname, ".") + // Look for FQDN of a known gateway hostname. + // Example: given "dist.ipfs.tech.ipns.dweb.link": + // 1. Lookup "link" TLD in knownGateways: negative + // 2. Lookup "dweb.link" in knownGateways: positive + // + // Stops when we have 2 or fewer labels left as we need at least a + // rootId and a namespace. + for i := len(labels) - 1; i >= 2; i-- { + fqdn := strings.Join(labels[i:], ".") + gw, ok := gws.isKnownHostname(fqdn) + if !ok { + continue + } + + ns := labels[i-1] + if !isSubdomainNamespace(ns) { + continue + } + + // Merge remaining labels (could be a FQDN with DNSLink) + rootID := strings.Join(labels[:i-1], ".") + return gw, fqdn, ns, rootID, true + } + // no match + return nil, "", "", "", false +} diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go new file mode 100644 index 000000000..f4ce73241 --- /dev/null +++ b/gateway/hostname_test.go @@ -0,0 +1,299 @@ +package gateway + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + cid "github.com/ipfs/go-cid" + path "github.com/ipfs/go-path" +) + +func TestToSubdomainURL(t *testing.T) { + gwAPI := newMockApi() + testCID, err := cid.Decode("bafkqaglimvwgy3zakrsxg5cun5jxkyten5wwc2lokvjeycq") + if err != nil { + t.Fatal(err) + } + + gwAPI.ns["/ipns/dnslink.long-name.example.com"] = path.FromString(testCID.String()) + gwAPI.ns["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = path.FromString(testCID.String()) + httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) + httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) + httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) + httpsProxiedRequest.Header.Set("X-Forwarded-Proto", "https") + + for _, test := range []struct { + // in: + request *http.Request + gwHostname string + inlineDNSLink bool + path string + // out: + url string + err error + }{ + + // DNSLink + {httpRequest, "localhost", false, "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil}, + // Hostname with port + {httpRequest, "localhost:8080", false, "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil}, + // CIDv0 → CIDv1base32 + {httpRequest, "localhost", false, "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil}, + // CIDv1 with long sha512 + {httpRequest, "localhost", false, "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, + // PeerID as CIDv1 needs to have libp2p-key multicodec + {httpRequest, "localhost", false, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://k2k4r8n0flx3ra0y5dr8fmyvwbzy3eiztmtq6th694k5a3rznayp3e4o.ipns.localhost/", nil}, + {httpRequest, "localhost", false, "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil}, + // PeerID: ed25519+identity multihash → CIDv1Base36 + {httpRequest, "localhost", false, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil}, + {httpRequest, "sub.localhost", false, "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.sub.localhost/", nil}, + // HTTPS requires DNSLink name to fit in a single DNS label – see "Option C" from https://github.com/ipfs/in-web-browsers/issues/169 + {httpRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "http://dnslink.long-name.example.com.ipns.dweb.link/", nil}, + {httpsRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "https://dnslink-long--name-example-com.ipns.dweb.link/", nil}, + {httpsProxiedRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "https://dnslink-long--name-example-com.ipns.dweb.link/", nil}, + // HTTP requests can also be converted to fit into a single DNS label - https://github.com/ipfs/kubo/issues/9243 + {httpRequest, "localhost", true, "/ipns/dnslink.long-name.example.com", "http://dnslink-long--name-example-com.ipns.localhost/", nil}, + {httpRequest, "dweb.link", true, "/ipns/dnslink.long-name.example.com", "http://dnslink-long--name-example-com.ipns.dweb.link/", nil}, + } { + + url, err := toSubdomainURL(test.gwHostname, test.path, test.request, test.inlineDNSLink, gwAPI) + if url != test.url || !equalError(err, test.err) { + t.Errorf("(%s, %v, %s) returned (%s, %v), expected (%s, %v)", test.gwHostname, test.inlineDNSLink, test.path, url, err, test.url, test.err) + } + } +} + +func TestToDNSLinkDNSLabel(t *testing.T) { + for _, test := range []struct { + in string + out string + err error + }{ + {"dnslink.long-name.example.com", "dnslink-long--name-example-com", nil}, + {"dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com", "", errors.New("DNSLink representation incompatible with DNS label length limit of 63: dnslink-too--long-f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o-example-com")}, + } { + out, err := toDNSLinkDNSLabel(test.in) + if out != test.out || !equalError(err, test.err) { + t.Errorf("(%s) returned (%s, %v), expected (%s, %v)", test.in, out, err, test.out, test.err) + } + } +} + +func TestToDNSLinkFQDN(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"singlelabel", "singlelabel"}, + {"docs-ipfs-tech", "docs.ipfs.tech"}, + {"dnslink-long--name-example-com", "dnslink.long-name.example.com"}, + } { + out := toDNSLinkFQDN(test.in) + if out != test.out { + t.Errorf("(%s) returned (%s), expected (%s)", test.in, out, test.out) + } + } +} + +func TestIsHTTPSRequest(t *testing.T) { + httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) + httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) + httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) + httpsProxiedRequest.Header.Set("X-Forwarded-Proto", "https") + httpProxiedRequest := httptest.NewRequest("GET", "http://proxied-http-request-stub.example.com", nil) + httpProxiedRequest.Header.Set("X-Forwarded-Proto", "http") + oddballRequest := httptest.NewRequest("GET", "foo://127.0.0.1:8080", nil) + for _, test := range []struct { + in *http.Request + out bool + }{ + {httpRequest, false}, + {httpsRequest, true}, + {httpsProxiedRequest, true}, + {httpProxiedRequest, false}, + {oddballRequest, false}, + } { + out := isHTTPSRequest(test.in) + if out != test.out { + t.Errorf("(%+v): returned %t, expected %t", test.in, out, test.out) + } + } +} + +func TestHasPrefix(t *testing.T) { + for _, test := range []struct { + prefixes []string + path string + out bool + }{ + {[]string{"/ipfs"}, "/ipfs/cid", true}, + {[]string{"/ipfs/"}, "/ipfs/cid", true}, + {[]string{"/version/"}, "/version", true}, + {[]string{"/version"}, "/version", true}, + } { + out := hasPrefix(test.path, test.prefixes...) + if out != test.out { + t.Errorf("(%+v, %s) returned '%t', expected '%t'", test.prefixes, test.path, out, test.out) + } + } +} + +func TestIsDomainNameAndNotPeerID(t *testing.T) { + for _, test := range []struct { + hostname string + out bool + }{ + {"", false}, + {"example.com", true}, + {"non-icann.something", true}, + {"..", false}, + {"12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", false}, // valid peerid + {"k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna", false}, // valid peerid + } { + out := isDomainNameAndNotPeerID(test.hostname) + if out != test.out { + t.Errorf("(%s) returned '%t', expected '%t'", test.hostname, out, test.out) + } + } +} + +func TestPortStripping(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"localhost:8080", "localhost"}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost"}, + {"example.com:443", "example.com"}, + {"example.com", "example.com"}, + {"foo-dweb.ipfs.pvt.k12.ma.us:8080", "foo-dweb.ipfs.pvt.k12.ma.us"}, + {"localhost", "localhost"}, + {"[::1]:8080", "::1"}, + } { + out := stripPort(test.in) + if out != test.out { + t.Errorf("(%s): returned '%s', expected '%s'", test.in, out, test.out) + } + } +} + +func TestToDNSLabel(t *testing.T) { + for _, test := range []struct { + in string + out string + err error + }{ + // <= 63 + {"QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", nil}, + {"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", nil}, + // > 63 + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "k51qzi5uqu5dj16qyiq0tajolkojyl9qdkr254920wxv7ghtuwcz593tp69z9m", nil}, + // CIDv1 with long sha512 → error + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, + } { + inCID, _ := cid.Decode(test.in) + out, err := toDNSLabel(test.in, inCID) + if out != test.out || !equalError(err, test.err) { + t.Errorf("(%s): returned (%s, %v) expected (%s, %v)", test.in, out, err, test.out, test.err) + } + } + +} + +func TestKnownSubdomainDetails(t *testing.T) { + gwLocalhost := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwDweb := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwLong := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwWildcard1 := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwWildcard2 := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + + gateways := prepareHostnameGateways(map[string]*Specification{ + "localhost": gwLocalhost, + "dweb.link": gwDweb, + "devgateway.dweb.link": gwDweb, + "dweb.ipfs.pvt.k12.ma.us": gwLong, // note the sneaky ".ipfs." ;-) + "*.wildcard1.tld": gwWildcard1, + "*.*.wildcard2.tld": gwWildcard2, + }) + + for _, test := range []struct { + // in: + hostHeader string + // out: + gw *Specification + hostname string + ns string + rootID string + ok bool + }{ + // no subdomain + {"127.0.0.1:8080", nil, "", "", "", false}, + {"[::1]:8080", nil, "", "", "", false}, + {"hey.look.example.com", nil, "", "", "", false}, + {"dweb.link", nil, "", "", "", false}, + // malformed Host header + {".....dweb.link", nil, "", "", "", false}, + {"link", nil, "", "", "", false}, + {"8080:dweb.link", nil, "", "", "", false}, + {" ", nil, "", "", "", false}, + {"", nil, "", "", "", false}, + // unknown gateway host + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", nil, "", "", "", false}, + // cid in subdomain, known gateway + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", gwLocalhost, "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.devgateway.dweb.link", gwDweb, "devgateway.dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + // capture everything before .ipfs. + {"foo.bar.boo-buzz.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, + // ipns + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // edge case check: public gateway under long TLD (see: https://publicsuffix.org) + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // dnslink in subdomain + {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost", gwLocalhost, "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"dist.ipfs.tech.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "dist.ipfs.tech", true}, + {"en.wikipedia-on-ipfs.org.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, + // edge case check: public gateway under long TLD (see: https://publicsuffix.org) + {"foo.dweb.ipfs.pvt.k12.ma.us", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // other namespaces + {"api.localhost", nil, "", "", "", false}, + {"peerid.p2p.localhost", gwLocalhost, "localhost", "p2p", "peerid", true}, + // wildcards + {"wildcard1.tld", nil, "", "", "", false}, + {".wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub.wildcard1.tld", gwWildcard1, "sub.wildcard1.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard2.tld", gwWildcard2, "sub1.sub2.wildcard2.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + } { + gw, hostname, ns, rootID, ok := gateways.knownSubdomainDetails(test.hostHeader) + if ok != test.ok { + t.Errorf("knownSubdomainDetails(%s): ok is %t, expected %t", test.hostHeader, ok, test.ok) + } + if rootID != test.rootID { + t.Errorf("knownSubdomainDetails(%s): rootID is '%s', expected '%s'", test.hostHeader, rootID, test.rootID) + } + if ns != test.ns { + t.Errorf("knownSubdomainDetails(%s): ns is '%s', expected '%s'", test.hostHeader, ns, test.ns) + } + if hostname != test.hostname { + t.Errorf("knownSubdomainDetails(%s): hostname is '%s', expected '%s'", test.hostHeader, hostname, test.hostname) + } + if gw != test.gw { + t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, test.gw) + } + } + +} + +func equalError(a, b error) bool { + return (a == nil && b == nil) || (a != nil && b != nil && a.Error() == b.Error()) +} diff --git a/go.mod b/go.mod index cd213febb..9dc88453c 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 + github.com/ipfs/go-namesys v0.7.0 github.com/ipfs/go-path v0.3.0 github.com/ipfs/go-peertaskqueue v0.8.0 github.com/ipfs/interface-go-ipfs-core v0.10.0 @@ -38,6 +39,7 @@ require ( github.com/libp2p/go-libp2p-record v0.2.0 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-msgio v0.2.0 + github.com/miekg/dns v1.1.50 github.com/multiformats/go-multiaddr v0.8.0 github.com/multiformats/go-multibase v0.1.1 github.com/multiformats/go-multicodec v0.6.0 @@ -63,6 +65,8 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gopacket v1.1.19 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -81,13 +85,14 @@ require ( github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.19.0 // indirect + github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-netroute v0.2.0 // indirect github.com/libp2p/go-openssl v0.1.0 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/miekg/dns v1.1.50 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -106,7 +111,9 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect + github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0 // indirect + github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect go.uber.org/atomic v1.10.0 // indirect golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect diff --git a/go.sum b/go.sum index 98732459b..ee49196ca 100644 --- a/go.sum +++ b/go.sum @@ -309,10 +309,14 @@ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmv github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -460,6 +464,8 @@ github.com/ipfs/go-merkledag v0.9.0 h1:DFC8qZ96Dz1hMT7dtIpcY524eFFDiEWAF8hNJHWW2 github.com/ipfs/go-merkledag v0.9.0/go.mod h1:bPHqkHt5OZ0p1n3iqPeDiw2jIBkjAytRjS3WSBwjq90= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-namesys v0.7.0 h1:xqosk71GIVRkFDtF2UNRcXn4LdNeo7tzuy8feHD6NbU= +github.com/ipfs/go-namesys v0.7.0/go.mod h1:KYSZBVZG3VJC34EfqqJPG7T48aWgxseoMPAPA5gLyyQ= github.com/ipfs/go-path v0.3.0 h1:tkjga3MtpXyM5v+3EbRvOHEoo+frwi4oumw5K+KYWyA= github.com/ipfs/go-path v0.3.0/go.mod h1:NOScsVgxfC/eIw4nz6OiGwK42PjaSJ4Y/ZFPn1Xe07I= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= @@ -609,6 +615,10 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= +github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= +github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= +github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= +github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= @@ -1068,10 +1078,13 @@ github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvS github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= +github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0 h1:obKzQ1ey5AJg5NKjgtTo/CKwLImVP4ETLRcsmzFJ4Qw= github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=