diff --git a/examples/gateway/car-file-gateway/libp2pio.car b/examples/gateway/car-file-gateway/libp2pio.car new file mode 100644 index 000000000..6514eef6d Binary files /dev/null and b/examples/gateway/car-file-gateway/libp2pio.car differ diff --git a/examples/gateway/car/main.go b/examples/gateway/car/main.go index 4ba63d253..292922c60 100644 --- a/examples/gateway/car/main.go +++ b/examples/gateway/car/main.go @@ -29,7 +29,7 @@ func main() { } defer f.Close() - gwAPI, err := common.NewBlocksGateway(blockService, nil) + gwAPI, err := gateway.NewBlocksGateway(blockService) if err != nil { log.Fatal(err) } diff --git a/examples/gateway/car/main_test.go b/examples/gateway/car/main_test.go index aebf0b997..dd2c835d3 100644 --- a/examples/gateway/car/main_test.go +++ b/examples/gateway/car/main_test.go @@ -1,6 +1,7 @@ package main import ( + "github.com/ipfs/go-libipfs/gateway" "io" "net/http" "net/http/httptest" @@ -22,7 +23,7 @@ func newTestServer() (*httptest.Server, io.Closer, error) { return nil, nil, err } - gateway, err := common.NewBlocksGateway(blockService, nil) + gateway, err := gateway.NewBlocksGateway(blockService) if err != nil { _ = f.Close() return nil, nil, err diff --git a/examples/gateway/common/blocks.go b/examples/gateway/common/blocks.go index a0a0f4702..ee0419768 100644 --- a/examples/gateway/common/blocks.go +++ b/examples/gateway/common/blocks.go @@ -1,42 +1,12 @@ package common import ( - "context" - "errors" - "fmt" "net/http" - gopath "path" - "github.com/ipfs/go-blockservice" - "github.com/ipfs/go-cid" - bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" - blockstore "github.com/ipfs/go-ipfs-blockstore" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" - "github.com/ipfs/go-libipfs/files" "github.com/ipfs/go-libipfs/gateway" - "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-namesys" - "github.com/ipfs/go-namesys/resolve" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/go-path/resolver" - "github.com/ipfs/go-unixfs" - ufile "github.com/ipfs/go-unixfs/file" - uio "github.com/ipfs/go-unixfs/io" - "github.com/ipfs/go-unixfsnode" - iface "github.com/ipfs/interface-go-ipfs-core" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - ifacepath "github.com/ipfs/interface-go-ipfs-core/path" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" - mc "github.com/multiformats/go-multicodec" ) -func NewBlocksHandler(gw *BlocksGateway, port int) http.Handler { +func NewBlocksHandler(gw gateway.IPFSBackend, port int) http.Handler { headers := map[string][]string{} gateway.AddAccessControlHeaders(headers) @@ -50,202 +20,3 @@ func NewBlocksHandler(gw *BlocksGateway, port int) http.Handler { mux.Handle("/ipns/", gwHandler) return mux } - -type BlocksGateway struct { - blockStore blockstore.Blockstore - blockService blockservice.BlockService - dagService format.DAGService - resolver resolver.Resolver - - // Optional routing system to handle /ipns addresses. - namesys namesys.NameSystem - routing routing.ValueStore -} - -func NewBlocksGateway(blockService blockservice.BlockService, routing routing.ValueStore) (*BlocksGateway, error) { - // Setup the DAG services, which use the CAR block store. - dagService := merkledag.NewDAGService(blockService) - - // Setup the UnixFS resolver. - fetcherConfig := bsfetcher.NewFetcherConfig(blockService) - fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) - resolver := resolver.NewBasicResolver(fetcher) - - // Setup a name system so that we are able to resolve /ipns links. - var ( - ns namesys.NameSystem - err error - ) - if routing != nil { - ns, err = namesys.NewNameSystem(routing) - if err != nil { - return nil, err - } - } - - return &BlocksGateway{ - blockStore: blockService.Blockstore(), - blockService: blockService, - dagService: dagService, - resolver: resolver, - routing: routing, - namesys: ns, - }, nil -} - -func (api *BlocksGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) { - nd, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } - - return ufile.NewUnixfsFile(ctx, api.dagService, nd) -} - -func (api *BlocksGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) { - node, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } - - dir, err := uio.NewDirectoryFromNode(api.dagService, node) - if err != nil { - return nil, err - } - - out := make(chan iface.DirEntry, uio.DefaultShardWidth) - - go func() { - defer close(out) - for l := range dir.EnumLinksAsync(ctx) { - select { - case out <- api.processLink(ctx, l): - case <-ctx.Done(): - return - } - } - }() - - return out, nil -} - -func (api *BlocksGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return api.blockService.GetBlock(ctx, c) -} - -func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - if api.routing == nil { - return nil, routing.ErrNotSupported - } - - // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful - // round trips to the remote routing provider. - if mc.Code(c.Type()) != mc.Libp2pKey { - return nil, errors.New("provided cid is not an encoded libp2p key") - } - - // The value store expects the key itself to be encoded as a multihash. - id, err := peer.FromCid(c) - if err != nil { - return nil, err - } - - return api.routing.GetValue(ctx, "/ipns/"+string(id)) -} - -func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { - if api.namesys != nil { - p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) - if err == namesys.ErrResolveRecursion { - err = nil - } - return ifacepath.New(p.String()), err - } - - return nil, errors.New("not implemented") -} - -func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool { - rp, err := api.ResolvePath(ctx, p) - if err != nil { - return false - } - - has, _ := api.blockStore.Has(ctx, rp.Cid()) - return has -} - -func (api *BlocksGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) { - if _, ok := p.(ifacepath.Resolved); ok { - return p.(ifacepath.Resolved), nil - } - - err := p.IsValid() - if err != nil { - return nil, err - } - - ipath := ipfspath.Path(p.String()) - if ipath.Segments()[0] == "ipns" { - ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) - if err != nil { - return nil, err - } - } - - if ipath.Segments()[0] != "ipfs" { - return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) - } - - node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath) - if err != nil { - return nil, err - } - - root, err := cid.Parse(ipath.Segments()[1]) - if err != nil { - return nil, err - } - - return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil -} - -func (api *BlocksGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) { - rp, err := api.ResolvePath(ctx, p) - if err != nil { - return nil, err - } - - node, err := api.dagService.Get(ctx, rp.Cid()) - if err != nil { - return nil, fmt.Errorf("get node: %w", err) - } - return node, nil -} - -func (api *BlocksGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry { - if result.Err != nil { - return iface.DirEntry{Err: result.Err} - } - - link := iface.DirEntry{ - Name: result.Link.Name, - Cid: result.Link.Cid, - } - - switch link.Cid.Type() { - case cid.Raw: - link.Type = iface.TFile - link.Size = result.Link.Size - case cid.DagProtobuf: - link.Size = result.Link.Size - } - - return link -} diff --git a/examples/gateway/proxy/main.go b/examples/gateway/proxy/main.go index 793fee121..56eb94231 100644 --- a/examples/gateway/proxy/main.go +++ b/examples/gateway/proxy/main.go @@ -27,7 +27,7 @@ func main() { routing := newProxyRouting(*gatewayUrlPtr, nil) // Creates the gateway with the block service and the routing. - gwAPI, err := common.NewBlocksGateway(blockService, routing) + gwAPI, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing)) if err != nil { log.Fatal(err) } diff --git a/examples/gateway/proxy/main_test.go b/examples/gateway/proxy/main_test.go index 453f3e21b..10b1406bc 100644 --- a/examples/gateway/proxy/main_test.go +++ b/examples/gateway/proxy/main_test.go @@ -1,6 +1,7 @@ package main import ( + "github.com/ipfs/go-libipfs/gateway" "io" "net/http" "net/http/httptest" @@ -22,7 +23,7 @@ func newProxyGateway(t *testing.T, rs *httptest.Server) *httptest.Server { blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) routing := newProxyRouting(rs.URL, nil) - gateway, err := common.NewBlocksGateway(blockService, routing) + gateway, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing)) if err != nil { t.Error(err) } diff --git a/examples/go.mod b/examples/go.mod index c059028cc..f54d28033 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -7,21 +7,14 @@ require ( github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-fetcher v1.6.1 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-exchange-offline v0.3.0 - github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-ipns v0.3.0 github.com/ipfs/go-libipfs v0.4.0 github.com/ipfs/go-merkledag v0.9.0 - github.com/ipfs/go-namesys v0.7.0 - github.com/ipfs/go-path v0.3.1 - github.com/ipfs/go-unixfs v0.4.3 - github.com/ipfs/go-unixfsnode v1.5.2 - github.com/ipfs/interface-go-ipfs-core v0.10.0 + github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa github.com/ipld/go-car/v2 v2.6.0 - github.com/ipld/go-codec-dagpb v1.5.0 github.com/ipld/go-ipld-prime v0.19.0 github.com/libp2p/go-libp2p v0.25.1 github.com/libp2p/go-libp2p-routing-helpers v0.6.0 @@ -67,6 +60,7 @@ require ( github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.1.1 // indirect + github.com/ipfs/go-fetcher v1.6.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect @@ -75,13 +69,19 @@ require ( github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect + github.com/ipfs/go-ipld-format v0.4.0 // indirect github.com/ipfs/go-ipld-legacy v0.1.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/ipfs/go-namesys v0.7.0 // indirect + github.com/ipfs/go-path v0.3.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect + github.com/ipfs/go-unixfsnode v1.5.2 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect + github.com/ipfs/interface-go-ipfs-core v0.10.0 // indirect github.com/ipld/go-car v0.5.0 // indirect + github.com/ipld/go-codec-dagpb v1.5.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -90,6 +90,7 @@ require ( github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect + github.com/libp2p/go-doh-resolver v0.4.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.21.0 // indirect diff --git a/examples/go.sum b/examples/go.sum index d53d3aea5..14063a3f7 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -481,8 +481,8 @@ github.com/ipfs/go-path v0.3.1/go.mod h1:eNLsxJEEMxn/CDzUJ6wuNl+6No6tEUhOZcPKsZs github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= -github.com/ipfs/go-unixfs v0.4.3 h1:EdDc1sNZNFDUlo4UrVAvvAofVI5EwTnKu8Nv8mgXkWQ= -github.com/ipfs/go-unixfs v0.4.3/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= +github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa h1:X8DPpsI3xvdsNxrsHi+ji39rjIvfPna3+XD+iQehbNQ= +github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew= github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= @@ -576,6 +576,8 @@ github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0 github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU= github.com/libp2p/go-conn-security-multistream v0.2.1/go.mod h1:cR1d8gA0Hr59Fj6NhaTpFhJZrjSYuNmhpT2r25zYR70= +github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw= +github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVhhBjyhhCJs8= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= @@ -840,6 +842,7 @@ github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6 github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= +github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= diff --git a/files/readerfile.go b/files/readerfile.go index a03dae23f..7b4e07954 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -18,7 +18,18 @@ type ReaderFile struct { } func NewBytesFile(b []byte) File { - return &ReaderFile{"", NewReaderFile(bytes.NewReader(b)), nil, int64(len(b))} + return &ReaderFile{"", bytesReaderCloser{bytes.NewReader(b)}, nil, int64(len(b))} +} + +// TODO: Is this the best way to fix this bug? +// The bug is we want to be an io.ReadSeekCloser, but bytes.NewReader only gives a io.ReadSeeker and io.NopCloser +// effectively removes the io.Seeker ability from the passed in io.Reader +type bytesReaderCloser struct { + *bytes.Reader +} + +func (b bytesReaderCloser) Close() error { + return nil } func NewReaderFile(reader io.Reader) File { diff --git a/gateway/blocks_gateway.go b/gateway/blocks_gateway.go new file mode 100644 index 000000000..776e855bb --- /dev/null +++ b/gateway/blocks_gateway.go @@ -0,0 +1,468 @@ +package gateway + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + gopath "path" + "strings" + + "go.uber.org/multierr" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" + blockstore "github.com/ipfs/go-ipfs-blockstore" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-namesys" + "github.com/ipfs/go-namesys/resolve" + ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/go-path/resolver" + ufile "github.com/ipfs/go-unixfs/file" + uio "github.com/ipfs/go-unixfs/io" + "github.com/ipfs/go-unixfsnode" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" + ifacepath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipld/go-car" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + mc "github.com/multiformats/go-multicodec" +) + +type BlocksGateway struct { + blockStore blockstore.Blockstore + blockService blockservice.BlockService + dagService format.DAGService + resolver resolver.Resolver + + // Optional routing system to handle /ipns addresses. + namesys namesys.NameSystem + routing routing.ValueStore +} + +var _ IPFSBackend = (*BlocksGateway)(nil) + +type gwOptions struct { + ns namesys.NameSystem + vs routing.ValueStore +} + +// WithNameSystem sets the name system to use for the gateway. If not set it will use a default DNSLink resolver +// along with any configured ValueStore +func WithNameSystem(ns namesys.NameSystem) BlockGatewayOption { + return func(opts *gwOptions) error { + opts.ns = ns + return nil + } +} + +// WithValueStore sets the ValueStore to use for the gateway +func WithValueStore(vs routing.ValueStore) BlockGatewayOption { + return func(opts *gwOptions) error { + opts.vs = vs + return nil + } +} + +type BlockGatewayOption func(gwOptions *gwOptions) error + +func NewBlocksGateway(blockService blockservice.BlockService, opts ...BlockGatewayOption) (*BlocksGateway, error) { + var compiledOptions gwOptions + for _, o := range opts { + if err := o(&compiledOptions); err != nil { + return nil, err + } + } + + // Setup the DAG services, which use the CAR block store. + dagService := merkledag.NewDAGService(blockService) + + // Setup the UnixFS resolver. + fetcherConfig := bsfetcher.NewFetcherConfig(blockService) + fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) + r := resolver.NewBasicResolver(fetcher) + + // Setup a name system so that we are able to resolve /ipns links. + var ( + ns namesys.NameSystem + vs routing.ValueStore + ) + + vs = compiledOptions.vs + if vs == nil { + vs = routinghelpers.Null{} + } + + ns = compiledOptions.ns + if ns == nil { + dns, err := NewDNSResolver(nil, nil) + if err != nil { + return nil, err + } + + ns, err = namesys.NewNameSystem(vs, namesys.WithDNSResolver(dns)) + if err != nil { + return nil, err + } + } + + return &BlocksGateway{ + blockStore: blockService.Blockstore(), + blockService: blockService, + dagService: dagService, + resolver: r, + routing: vs, + namesys: ns, + }, nil +} + +func (api *BlocksGateway) Get(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *GetResponse, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + rootCodec := nd.Cid().Prefix().GetCodec() + // This covers both Raw blocks and terminal IPLD codecs like dag-cbor and dag-json + // Note: while only cbor, json, dag-cbor, and dag-json are currently supported by gateways this could change + if rootCodec != uint64(mc.DagPb) { + return md, NewGetResponseFromFile(files.NewBytesFile(nd.RawData())), nil + } + + // This code path covers full graph, single file/directory, and range requests + f, err := ufile.NewUnixfsFile(ctx, api.dagService, nd) + // Note: there is an assumption here that non-UnixFS dag-pb should not be returned which is currently valid + if err != nil { + return md, nil, err + } + + if d, ok := f.(files.Directory); ok { + dir, err := uio.NewDirectoryFromNode(api.dagService, nd) + if err != nil { + return md, nil, err + } + sz, err := d.Size() + if err != nil { + return ContentPathMetadata{}, nil, fmt.Errorf("could not get cumulative directory graph size: %w", err) + } + if sz < 0 { + return ContentPathMetadata{}, nil, fmt.Errorf("directory cumulative graph size cannot be negative") + } + return md, NewGetResponseFromDirectoryListing(uint64(sz), dir.EnumLinksAsync(ctx)), nil + } + if file, ok := f.(files.File); ok { + return md, NewGetResponseFromFile(file), nil + } + + return ContentPathMetadata{}, nil, fmt.Errorf("data was not a valid file or directory: %w", ErrInternalServerError) // TODO: should there be a gateway invalid content type to abstract over the various IPLD error types? +} + +func (api *BlocksGateway) GetRange(ctx context.Context, path ImmutablePath, ranges ...GetRange) (ContentPathMetadata, files.File, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + // This code path covers full graph, single file/directory, and range requests + n, err := ufile.NewUnixfsFile(ctx, api.dagService, nd) + if err != nil { + return md, nil, err + } + f, ok := n.(files.File) + if !ok { + return ContentPathMetadata{}, nil, NewErrorResponse(fmt.Errorf("can only do range requests on files, but did not get a file"), http.StatusBadRequest) + } + + return md, f, nil +} + +func (api *BlocksGateway) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + // This code path covers full graph, single file/directory, and range requests + n, err := ufile.NewUnixfsFile(ctx, api.dagService, nd) + if err != nil { + return md, nil, err + } + return md, n, nil +} + +func (api *BlocksGateway) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + return md, files.NewBytesFile(nd.RawData()), nil +} + +func (api *BlocksGateway) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + md, nd, err := api.getNode(ctx, path) + if err != nil { + return md, nil, err + } + + rootCodec := nd.Cid().Prefix().GetCodec() + if rootCodec != uint64(mc.DagPb) { + return md, files.NewBytesFile(nd.RawData()), nil + } + + // TODO: We're not handling non-UnixFS dag-pb. There's a bit of a discrepancy between what we want from a HEAD request and a Resolve request here and we're using this for both + fileNode, err := ufile.NewUnixfsFile(ctx, api.dagService, nd) + if err != nil { + return ContentPathMetadata{}, nil, err + } + + return md, fileNode, nil +} + +func (api *BlocksGateway) GetCAR(ctx context.Context, path ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) { + // Same go-car settings as dag.export command + store := dagStore{api: api, ctx: ctx} + + // TODO: When switching to exposing path blocks we'll want to add these as well + roots, lastSeg, err := api.getPathRoots(ctx, path) + if err != nil { + return ContentPathMetadata{}, nil, nil, err + } + + md := ContentPathMetadata{ + PathSegmentRoots: roots, + LastSegment: lastSeg, + } + + rootCid := lastSeg.Cid() + + // TODO: support selectors passed as request param: https://github.com/ipfs/kubo/issues/8769 + // TODO: this is very slow if blocks are remote due to linear traversal. Do we need deterministic traversals here? + dag := car.Dag{Root: rootCid, Selector: selectorparse.CommonSelector_ExploreAllRecursively} + c := car.NewSelectiveCar(ctx, store, []car.Dag{dag}, car.TraverseLinksOnlyOnce()) + r, w := io.Pipe() + + errCh := make(chan error, 1) + go func() { + carWriteErr := c.Write(w) + pipeCloseErr := w.Close() + errCh <- multierr.Combine(carWriteErr, pipeCloseErr) + close(errCh) + }() + + return md, r, errCh, nil +} + +func (api *BlocksGateway) getNode(ctx context.Context, path ImmutablePath) (ContentPathMetadata, format.Node, error) { + roots, lastSeg, err := api.getPathRoots(ctx, path) + if err != nil { + return ContentPathMetadata{}, nil, err + } + + md := ContentPathMetadata{ + PathSegmentRoots: roots, + LastSegment: lastSeg, + } + + lastRoot := lastSeg.Cid() + + nd, err := api.dagService.Get(ctx, lastRoot) + if err != nil { + return ContentPathMetadata{}, nil, err + } + + return md, nd, err +} + +func (api *BlocksGateway) getPathRoots(ctx context.Context, contentPath ImmutablePath) ([]cid.Cid, ifacepath.Resolved, error) { + /* + These are logical roots where each CID represent one path segment + and resolves to either a directory or the root block of a file. + The main purpose of this header is allow HTTP caches to do smarter decisions + around cache invalidation (eg. keep specific subdirectory/file if it did not change) + A good example is Wikipedia, which is HAMT-sharded, but we only care about + logical roots that represent each segment of the human-readable content + path: + Given contentPath = /ipns/en.wikipedia-on-ipfs.org/wiki/Block_of_Wikipedia_in_Turkey + rootCidList is a generated by doing `ipfs resolve -r` on each sub path: + /ipns/en.wikipedia-on-ipfs.org → bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze + /ipns/en.wikipedia-on-ipfs.org/wiki/ → bafybeihn2f7lhumh4grizksi2fl233cyszqadkn424ptjajfenykpsaiw4 + /ipns/en.wikipedia-on-ipfs.org/wiki/Block_of_Wikipedia_in_Turkey → bafkreibn6euazfvoghepcm4efzqx5l3hieof2frhp254hio5y7n3hv5rma + The result is an ordered array of values: + X-Ipfs-Roots: bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze,bafybeihn2f7lhumh4grizksi2fl233cyszqadkn424ptjajfenykpsaiw4,bafkreibn6euazfvoghepcm4efzqx5l3hieof2frhp254hio5y7n3hv5rma + Note that while the top one will change every time any article is changed, + the last root (responsible for specific article) may not change at all. + */ + var sp strings.Builder + var pathRoots []cid.Cid + contentPathStr := contentPath.String() + pathSegments := strings.Split(contentPathStr[6:], "/") + sp.WriteString(contentPathStr[:5]) // /ipfs or /ipns + var lastPath ifacepath.Resolved + for _, root := range pathSegments { + if root == "" { + continue + } + sp.WriteString("/") + sp.WriteString(root) + resolvedSubPath, err := api.resolvePath(ctx, ifacepath.New(sp.String())) + if err != nil { + // TODO: should we be more explicit here and is this part of the Gateway API contract? + // The issue here was that we returned datamodel.ErrWrongKind instead of this resolver error + if isErrNotFound(err) { + return nil, nil, resolver.ErrNoLink{Name: root, Node: lastPath.Cid()} + } + return nil, nil, err + } + lastPath = resolvedSubPath + pathRoots = append(pathRoots, lastPath.Cid()) + } + + pathRoots = pathRoots[:len(pathRoots)-1] + return pathRoots, lastPath, nil +} + +// FIXME(@Jorropo): https://github.com/ipld/go-car/issues/315 +type dagStore struct { + api *BlocksGateway + ctx context.Context +} + +func (ds dagStore) Get(_ context.Context, c cid.Cid) (blocks.Block, error) { + return ds.api.blockService.GetBlock(ds.ctx, c) +} + +func (api *BlocksGateway) ResolveMutable(ctx context.Context, p ifacepath.Path) (ImmutablePath, error) { + err := p.IsValid() + if err != nil { + return ImmutablePath{}, err + } + + ipath := ipfspath.Path(p.String()) + switch ipath.Segments()[0] { + case "ipns": + ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) + if err != nil { + return ImmutablePath{}, err + } + imPath, err := NewImmutablePath(ifacepath.New(ipath.String())) + if err != nil { + return ImmutablePath{}, err + } + return imPath, nil + case "ipfs": + imPath, err := NewImmutablePath(ifacepath.New(ipath.String())) + if err != nil { + return ImmutablePath{}, err + } + return imPath, nil + default: + return ImmutablePath{}, NewErrorResponse(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + } +} + +func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + if api.routing == nil { + return nil, NewErrorResponse(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) + } + + // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful + // round trips to the remote routing provider. + if mc.Code(c.Type()) != mc.Libp2pKey { + return nil, NewErrorResponse(errors.New("cid codec must be libp2p-key"), http.StatusBadRequest) + } + + // The value store expects the key itself to be encoded as a multihash. + id, err := peer.FromCid(c) + if err != nil { + return nil, err + } + + return api.routing.GetValue(ctx, "/ipns/"+string(id)) +} + +func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { + if api.namesys != nil { + p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ifacepath.New(p.String()), err + } + + return nil, NewErrorResponse(errors.New("not implemented"), http.StatusNotImplemented) +} + +func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool { + rp, err := api.resolvePath(ctx, p) + if err != nil { + return false + } + + has, _ := api.blockStore.Has(ctx, rp.Cid()) + return has +} + +func (api *BlocksGateway) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { + roots, lastSeg, err := api.getPathRoots(ctx, path) + if err != nil { + return ContentPathMetadata{}, err + } + md := ContentPathMetadata{ + PathSegmentRoots: roots, + LastSegment: lastSeg, + } + return md, nil +} + +func (api *BlocksGateway) resolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) { + if _, ok := p.(ifacepath.Resolved); ok { + return p.(ifacepath.Resolved), nil + } + + err := p.IsValid() + if err != nil { + return nil, err + } + + ipath := ipfspath.Path(p.String()) + if ipath.Segments()[0] == "ipns" { + ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) + if err != nil { + return nil, err + } + } + + if ipath.Segments()[0] != "ipfs" { + return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) + } + + node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath) + if err != nil { + return nil, err + } + + root, err := cid.Parse(ipath.Segments()[1]) + if err != nil { + return nil, err + } + + return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil +} diff --git a/gateway/blocks_gateway_files.go b/gateway/blocks_gateway_files.go new file mode 100644 index 000000000..d3a9120a1 --- /dev/null +++ b/gateway/blocks_gateway_files.go @@ -0,0 +1,80 @@ +package gateway + +import ( + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-unixfs" +) + +type wrappedLink struct { + link *format.Link +} + +func (w *wrappedLink) Size() (int64, error) { + return int64(w.link.Size), nil +} + +func (w *wrappedLink) Cid() cid.Cid { + return w.link.Cid +} + +func (w *wrappedLink) Close() error { + return nil +} + +type wrappedDirLinkChan struct { + dir <-chan unixfs.LinkResult + err error + name string + node *wrappedLink +} + +func (it *wrappedDirLinkChan) Next() bool { + result, ok := <-it.dir + if !ok { + return false + } + + if result.Err != nil { + it.err = result.Err + return false + } + + it.name = result.Link.Name + it.node = &wrappedLink{ + link: result.Link, + } + return true +} + +func (it *wrappedDirLinkChan) Err() error { + return it.err +} + +func (it *wrappedDirLinkChan) Name() string { + return it.name +} + +func (e *wrappedDirLinkChan) Node() files.Node { + return e.node +} + +type wrappedDirectory struct { + node files.Node + dir <-chan unixfs.LinkResult +} + +func (d *wrappedDirectory) Close() error { + return d.node.Close() +} + +func (d *wrappedDirectory) Size() (int64, error) { + return d.node.Size() +} + +func (d *wrappedDirectory) Entries() files.DirIterator { + return &wrappedDirLinkChan{dir: d.dir} +} + +var _ files.Directory = &wrappedDirectory{} diff --git a/gateway/dns.go b/gateway/dns.go new file mode 100644 index 000000000..504bb8311 --- /dev/null +++ b/gateway/dns.go @@ -0,0 +1,92 @@ +package gateway + +import ( + "fmt" + "strings" + + "github.com/libp2p/go-doh-resolver" + dns "github.com/miekg/dns" + madns "github.com/multiformats/go-multiaddr-dns" +) + +var defaultResolvers = map[string]string{ + "eth.": "https://resolver.cloudflare-eth.com/dns-query", + "crypto.": "https://resolver.cloudflare-eth.com/dns-query", +} + +func newResolver(url string, opts ...doh.Option) (madns.BasicResolver, error) { + if !strings.HasPrefix(url, "https://") { + return nil, fmt.Errorf("invalid resolver url: %s", url) + } + + return doh.NewResolver(url, opts...) +} + +// NewDNSResolver creates a new DNS resolver based on the default resolvers and +// the provided resolvers. +// +// The argument 'resolvers' is a map of FQDNs to URLs for custom DNS resolution. +// URLs starting with `https://` indicate DoH endpoints. Support for other resolver +// types may be added in the future. +// +// https://en.wikipedia.org/wiki/Fully_qualified_domain_name +// https://en.wikipedia.org/wiki/DNS_over_HTTPS +// +// Example: +// - Custom resolver for ENS: `eth.` → `https://eth.link/dns-query` +// - Override the default OS resolver: `.` → `https://doh.applied-privacy.net/query` +func NewDNSResolver(resolvers map[string]string, dohOpts ...doh.Option) (*madns.Resolver, error) { + var opts []madns.Option + var err error + + domains := make(map[string]struct{}) // to track overridden default resolvers + rslvrs := make(map[string]madns.BasicResolver) // to reuse resolvers for the same URL + + for domain, url := range resolvers { + if domain != "." && !dns.IsFqdn(domain) { + return nil, fmt.Errorf("invalid domain %s; must be FQDN", domain) + } + + domains[domain] = struct{}{} + if url == "" { + // allow overriding of implicit defaults with the default resolver + continue + } + + rslv, ok := rslvrs[url] + if !ok { + rslv, err = newResolver(url, dohOpts...) + if err != nil { + return nil, fmt.Errorf("bad resolver for %s: %w", domain, err) + } + rslvrs[url] = rslv + } + + if domain != "." { + opts = append(opts, madns.WithDomainResolver(domain, rslv)) + } else { + opts = append(opts, madns.WithDefaultResolver(rslv)) + } + } + + // fill in defaults if not overridden by the user + for domain, url := range defaultResolvers { + _, ok := domains[domain] + if ok { + continue + } + + rslv, ok := rslvrs[url] + if !ok { + rslv, err = newResolver(url) + if err != nil { + return nil, fmt.Errorf("bad resolver for %s: %w", domain, err) + } + rslvrs[url] = rslv + } + + opts = append(opts, madns.WithDomainResolver(domain, rslv)) + } + + return madns.NewResolver(opts...) +} diff --git a/gateway/errors.go b/gateway/errors.go index 47f40cf5b..acb3dce1f 100644 --- a/gateway/errors.go +++ b/gateway/errors.go @@ -10,6 +10,7 @@ import ( ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-path/resolver" + "github.com/ipld/go-ipld-prime/datamodel" ) var ( @@ -157,15 +158,24 @@ func isErrNotFound(err error) bool { return true } - // Checks if err is a resolver.ErrNoLink. resolver.ErrNoLink does not implement - // the .Is interface and cannot be directly compared to. Therefore, errors.Is - // always returns false with it. + // Checks if err is of a type that does not implement the .Is interface and + // cannot be directly compared to. Therefore, errors.Is cannot be used. for { _, ok := err.(resolver.ErrNoLink) if ok { return true } + _, ok = err.(datamodel.ErrWrongKind) + if ok { + return true + } + + _, ok = err.(datamodel.ErrNotExists) + if ok { + return true + } + err = errors.Unwrap(err) if err == nil { return false diff --git a/gateway/gateway.go b/gateway/gateway.go index 977c6b960..329456a27 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -2,13 +2,14 @@ package gateway import ( "context" + "fmt" + "io" "net/http" "sort" - cid "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-cid" "github.com/ipfs/go-libipfs/files" - iface "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core/path" ) @@ -17,34 +18,132 @@ type Config struct { Headers map[string][]string } -// API defines the minimal set of API services required for a gateway handler. -type API interface { - // GetUnixFsNode returns a read-only handle to a file tree referenced by a path. - GetUnixFsNode(context.Context, path.Resolved) (files.Node, error) +// TODO: Is this what we want for ImmutablePath? +type ImmutablePath struct { + p path.Path +} + +func NewImmutablePath(p path.Path) (ImmutablePath, error) { + if p.Mutable() { + return ImmutablePath{}, fmt.Errorf("path cannot be mutable") + } + return ImmutablePath{p: p}, nil +} + +func (i ImmutablePath) String() string { + return i.p.String() +} + +func (i ImmutablePath) Namespace() string { + return i.p.Namespace() +} + +func (i ImmutablePath) Mutable() bool { + return false +} + +func (i ImmutablePath) IsValid() error { + return i.p.IsValid() +} + +var _ path.Path = (*ImmutablePath)(nil) + +type ContentPathMetadata struct { + PathSegmentRoots []cid.Cid + LastSegment path.Resolved + ContentType string // Only used for UnixFS requests +} + +// GetRange describes a range request within a UnixFS file. From and To mostly follow HTTP Range Request semantics. +// From >= 0 and To = nil: Get the file (From, Length) +// From >= 0 and To >= 0: Get the range (From, To) +// From >= 0 and To <0: Get the range (From, Length - To) +type GetRange struct { + From uint64 + To *int64 +} + +type GetResponse struct { + bytes files.File + directory *directoryResponse +} - // LsUnixFsDir returns the list of links in a directory. - LsUnixFsDir(context.Context, path.Resolved) (<-chan iface.DirEntry, error) +type directoryResponse struct { + size uint64 + directory <-chan unixfs.LinkResult +} + +func NewGetResponseFromFile(file files.File) *GetResponse { + return &GetResponse{bytes: file} +} + +func NewGetResponseFromDirectoryListing(cumulativeTreeSize uint64, dirEntries <-chan unixfs.LinkResult) *GetResponse { + return &GetResponse{directory: &directoryResponse{ + size: cumulativeTreeSize, + directory: dirEntries, + }} +} - // GetBlock return a block from a certain CID. - GetBlock(context.Context, cid.Cid) (blocks.Block, error) +// IPFSBackend is the required set of functionality used to implement the IPFS HTTP Gateway specification. +// To signal error types to the gateway code (so that not everything is a HTTP 500) return an error wrapped with NewErrorResponse. +// There are also some existing error types that the gateway code knows how to handle (e.g. context.DeadlineExceeded +// and various IPLD pathing related errors). +type IPFSBackend interface { + // Get returns a UnixFS file, UnixFS directory, or an IPLD block depending on what the path is that has been + // requested. Directories' files.DirEntry objects do not need to contain content, but must contain Name, + // Size, and Cid. + Get(context.Context, ImmutablePath) (ContentPathMetadata, *GetResponse, error) + + // GetRange returns a full UnixFS file object. Ranges passed in are advisory for pre-fetching data, however + // consumers of this function may require extra data beyond the passed ranges (e.g. the initial bit of the file + // might be used for content type sniffing even if only the end of the file is requested). + GetRange(context.Context, ImmutablePath, ...GetRange) (ContentPathMetadata, files.File, error) + + // GetAll returns a UnixFS file or directory depending on what the path is that has been requested. Directories should + // include all content recursively. + GetAll(context.Context, ImmutablePath) (ContentPathMetadata, files.Node, error) + + // GetBlock returns a single block of data + GetBlock(context.Context, ImmutablePath) (ContentPathMetadata, files.File, error) + + // Head returns a file or directory depending on what the path is that has been requested. + // For UnixFS files should return a file which has the correct file size and either returns the ContentType in ContentPathMetadata or + // enough data (e.g. 3kiB) such that the content type can be determined by sniffing. + // For all other data types returning just size information is sufficient + // TODO: give function more explicit return types + Head(context.Context, ImmutablePath) (ContentPathMetadata, files.Node, error) + + // ResolvePath resolves the path using UnixFS resolver. If the path does not + // exist due to a missing link, it should return an error of type: + // NewErrorResponse(fmt.Errorf("no link named %q under %s", name, cid), http.StatusNotFound) + ResolvePath(context.Context, ImmutablePath) (ContentPathMetadata, error) + + // GetCAR returns a CAR file for the given immutable path + // Returns an initial error if there was an issue before the CAR streaming begins as well as a channel with a single + // that may contain a single error for if any errors occur during the streaming. If there was an initial error the + // error channel is nil + // TODO: Make this function signature better + GetCAR(context.Context, ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) + + // IsCached returns whether or not the path exists locally. + IsCached(context.Context, path.Path) bool // GetIPNSRecord retrieves the best IPNS record for a given CID (libp2p-key) // from the routing system. GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) + // ResolveMutable takes a mutable path and resolves it into an immutable one. This means recursively resolving any + // DNSLink or IPNS records. + // + // For example, given a mapping from `/ipns/dnslink.tld -> /ipns/ipns-id/mydirectory` and `/ipns/ipns-id` to + // `/ipfs/some-cid`, the result of passing `/ipns/dnslink.tld/myfile` would be `/ipfs/some-cid/mydirectory/myfile`. + ResolveMutable(context.Context, path.Path) (ImmutablePath, error) + // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN. // Unlike ResolvePath, it does not perform recursive resolution. It only // checks for the existence of a DNSLink TXT record with path starting with // /ipfs/ or /ipns/ and returns the path as-is. GetDNSLinkRecord(context.Context, string) (path.Path, error) - - // IsCached returns whether or not the path exists locally. - IsCached(context.Context, path.Path) bool - - // ResolvePath resolves the path using UnixFS resolver. If the path does not - // exist due to a missing link, it should return an error of type: - // https://pkg.go.dev/github.com/ipfs/go-path@v0.3.0/resolver#ErrNoLink - ResolvePath(context.Context, path.Path) (path.Resolved, error) } // A helper function to clean up a set of headers: diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 2b9ef0c01..540546d46 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -8,36 +8,19 @@ import ( "net/http" "net/http/httptest" "os" - gopath "path" "regexp" "strings" "testing" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" - bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" - blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/files" - "github.com/ipfs/go-merkledag" "github.com/ipfs/go-namesys" - "github.com/ipfs/go-namesys/resolve" path "github.com/ipfs/go-path" - "github.com/ipfs/go-path/resolver" - "github.com/ipfs/go-unixfs" - ufile "github.com/ipfs/go-unixfs/file" - uio "github.com/ipfs/go-unixfs/io" - "github.com/ipfs/go-unixfsnode" - iface "github.com/ipfs/interface-go-ipfs-core" nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" ipath "github.com/ipfs/interface-go-ipfs-core/path" carblockstore "github.com/ipld/go-car/v2/blockstore" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/routing" "github.com/stretchr/testify/assert" @@ -88,13 +71,12 @@ func (m mockNamesys) GetResolver(subs string) (namesys.Resolver, bool) { } type mockAPI struct { - blockStore blockstore.Blockstore - blockService blockservice.BlockService - dagService format.DAGService - resolver resolver.Resolver - namesys mockNamesys + gw IPFSBackend + namesys mockNamesys } +var _ IPFSBackend = (*mockAPI)(nil) + func newMockAPI(t *testing.T) (*mockAPI, cid.Cid) { r, err := os.Open("./testdata/fixtures.car") assert.Nil(t, err) @@ -112,65 +94,45 @@ func newMockAPI(t *testing.T) (*mockAPI, cid.Cid) { assert.Len(t, cids, 1) blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) - dagService := merkledag.NewDAGService(blockService) - fetcherConfig := bsfetcher.NewFetcherConfig(blockService) - fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) - resolver := resolver.NewBasicResolver(fetcher) + n := mockNamesys{} + gwApi, err := NewBlocksGateway(blockService, WithNameSystem(n)) + if err != nil { + t.Fatal(err) + } return &mockAPI{ - blockStore: blockService.Blockstore(), - blockService: blockService, - dagService: dagService, - resolver: resolver, - namesys: mockNamesys{}, + gw: gwApi, + namesys: n, }, cids[0] } -func (api *mockAPI) GetUnixFsNode(ctx context.Context, p ipath.Resolved) (files.Node, error) { - nd, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } - - return ufile.NewUnixfsFile(ctx, api.dagService, nd) +func (api *mockAPI) Get(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, *GetResponse, error) { + return api.gw.Get(ctx, immutablePath) } -func (api *mockAPI) LsUnixFsDir(ctx context.Context, p ipath.Resolved) (<-chan iface.DirEntry, error) { - node, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } +func (api *mockAPI) GetRange(ctx context.Context, immutablePath ImmutablePath, ranges ...GetRange) (ContentPathMetadata, files.File, error) { + return api.gw.GetRange(ctx, immutablePath, ranges...) +} - dir, err := uio.NewDirectoryFromNode(api.dagService, node) - if err != nil { - return nil, err - } +func (api *mockAPI) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + return api.gw.GetAll(ctx, immutablePath) +} - out := make(chan iface.DirEntry, uio.DefaultShardWidth) +func (api *mockAPI) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { + return api.gw.GetBlock(ctx, immutablePath) +} - go func() { - defer close(out) - for l := range dir.EnumLinksAsync(ctx) { - select { - case out <- api.processLink(ctx, l): - case <-ctx.Done(): - return - } - } - }() +func (api *mockAPI) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + return api.gw.Head(ctx, immutablePath) +} - return out, nil +func (api *mockAPI) GetCAR(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) { + return api.gw.GetCAR(ctx, immutablePath) } -func (api *mockAPI) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return api.blockService.GetBlock(ctx, c) +func (api *mockAPI) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { + return api.gw.ResolveMutable(ctx, p) } func (api *mockAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { @@ -190,82 +152,33 @@ func (api *mockAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (ipat } func (api *mockAPI) IsCached(ctx context.Context, p ipath.Path) bool { - rp, err := api.ResolvePath(ctx, p) - if err != nil { - return false - } - - has, _ := api.blockStore.Has(ctx, rp.Cid()) - return has + return api.gw.IsCached(ctx, p) } -func (api *mockAPI) ResolvePath(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { - if _, ok := ip.(ipath.Resolved); ok { - return ip.(ipath.Resolved), nil - } - - err := ip.IsValid() - if err != nil { - return nil, err - } +func (api *mockAPI) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { + return api.gw.ResolvePath(ctx, immutablePath) +} - p := path.Path(ip.String()) - if p.Segments()[0] == "ipns" { - p, err = resolve.ResolveIPNS(ctx, api.namesys, p) +func (api *mockAPI) resolvePathNoRootsReturned(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { + var imPath ImmutablePath + var err error + if ip.Mutable() { + imPath, err = api.ResolveMutable(ctx, ip) + if err != nil { + return nil, err + } + } else { + imPath, err = NewImmutablePath(ip) if err != nil { return nil, err } } - if p.Segments()[0] != "ipfs" { - return nil, fmt.Errorf("unsupported path namespace: %s", ip.Namespace()) - } - - node, rest, err := api.resolver.ResolveToLastNode(ctx, p) - if err != nil { - return nil, err - } - - root, err := cid.Parse(p.Segments()[1]) - if err != nil { - return nil, err - } - - return ipath.NewResolvedPath(p, node, root, gopath.Join(rest...)), nil -} - -func (api *mockAPI) resolveNode(ctx context.Context, p ipath.Path) (format.Node, error) { - rp, err := api.ResolvePath(ctx, p) + md, err := api.ResolvePath(ctx, imPath) if err != nil { return nil, err } - - node, err := api.dagService.Get(ctx, rp.Cid()) - if err != nil { - return nil, fmt.Errorf("get node: %w", err) - } - return node, nil -} - -func (api *mockAPI) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry { - if result.Err != nil { - return iface.DirEntry{Err: result.Err} - } - - link := iface.DirEntry{ - Name: result.Link.Name, - Cid: result.Link.Cid, - } - - switch link.Cid.Type() { - case cid.Raw: - link.Type = iface.TFile - link.Size = result.Link.Size - case cid.DagProtobuf: - link.Size = result.Link.Size - } - - return link + return md.LastSegment, nil } func doWithoutRedirect(req *http.Request) (*http.Response, error) { @@ -288,7 +201,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, *mock return ts, api, root } -func newTestServer(t *testing.T, api API) *httptest.Server { +func newTestServer(t *testing.T, api IPFSBackend) *httptest.Server { config := Config{Headers: map[string][]string{}} AddAccessControlHeaders(config.Headers) @@ -316,7 +229,7 @@ func TestGatewayGet(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name(), "fnord")) + k, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name(), "fnord")) assert.Nil(t, err) api.namesys["/ipns/example.com"] = path.FromCid(k.Cid()) @@ -423,7 +336,7 @@ func TestIPNSHostnameRedirect(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) + k, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) assert.Nil(t, err) t.Logf("k: %s\n", k) @@ -478,14 +391,14 @@ func TestIPNSHostnameBacklinks(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) + k, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) assert.Nil(t, err) // create /ipns/example.net/foo/ - k2, err := api.ResolvePath(ctx, ipath.Join(k, "foo? #<'")) + k2, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(k, "foo? #<'")) assert.Nil(t, err) - k3, err := api.ResolvePath(ctx, ipath.Join(k, "foo? #<'/bar")) + k3, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(k, "foo? #<'/bar")) assert.Nil(t, err) t.Logf("k: %s\n", k) @@ -562,7 +475,7 @@ func TestPretty404(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) + k, err := api.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) assert.Nil(t, err) host := "example.net" diff --git a/gateway/handler.go b/gateway/handler.go index 92f88df6b..9c7134fe1 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "fmt" "html/template" "io" @@ -17,9 +18,7 @@ import ( cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" - coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" - mc "github.com/multiformats/go-multicodec" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -64,7 +63,7 @@ type redirectTemplateData struct { // (it serves requests like GET /ipfs/QmVRzPKPzNtSrEzBFm2UZfxmPAgnaLke4DMcerbsGGSaFe/link) type handler struct { config Config - api API + api IPFSBackend // generic metrics firstContentBlockGetMetric *prometheus.HistogramVec @@ -196,11 +195,11 @@ func newHistogramMetric(name string, help string) *prometheus.HistogramVec { // NewHandler returns an http.Handler that can act as a gateway to IPFS content // offlineApi is a version of the API that should not make network requests for missing data -func NewHandler(c Config, api API) http.Handler { +func NewHandler(c Config, api IPFSBackend) http.Handler { return newHandler(c, api) } -func newHandler(c Config, api API) *handler { +func newHandler(c Config, api IPFSBackend) *handler { i := &handler{ config: c, api: api, @@ -332,7 +331,7 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { return } - contentPath := ipath.New(r.URL.Path) + contentPath := newPath(ipath.New(r.URL.Path)) ctx := context.WithValue(r.Context(), ContentPathKey, contentPath) r = r.WithContext(ctx) @@ -352,34 +351,55 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat)) - resolvedPath, contentPath, ok := i.handlePathResolution(w, r, responseFormat, contentPath, logger) - if !ok { + i.addUserHeaders(w) // ok, _now_ write user's headers. + w.Header().Set("X-Ipfs-Path", contentPath.originalRequestedPath.String()) + + // TODO: Why did the previous code do path resolution, was that a bug? + // TODO: Does If-None-Match apply here? + if responseFormat == "application/vnd.ipfs.ipns-record" { + logger.Debugw("serving ipns record", "path", contentPath) + success := i.serveIpnsRecord(r.Context(), w, r, contentPath, begin, logger) + if success { + i.getMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) + } + return } - trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResolvedPath", resolvedPath.String())) - // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified - if inm := r.Header.Get("If-None-Match"); inm != "" { - pathCid := resolvedPath.Cid() - // need to check against both File and Dir Etag variants - // because this inexpensive check happens before we do any I/O - cidEtag := getEtag(r, pathCid) - dirEtag := getDirListingEtag(pathCid) - if etagMatch(inm, cidEtag, dirEtag) { - // Finish early if client already has a matching Etag - w.WriteHeader(http.StatusNotModified) + if contentPath.originalRequestedPath.Mutable() { + immutableContentPath, err := i.api.ResolveMutable(r.Context(), contentPath.originalRequestedPath) + if err != nil { + // Note: webError will replace http.StatusInternalServerError with a more appropriate error (e.g. StatusNotFound, StatusRequestTimeout, StatusServiceUnavailable, etc.) if necessary + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.originalRequestedPath.String()), err) + webError(w, err, http.StatusInternalServerError) + return + } + contentPath.immutableRequestedPath = immutableContentPath + } else { + immutableContentPath, err := NewImmutablePath(contentPath.originalRequestedPath) + if err != nil { + err = fmt.Errorf("path was expected to be immutable, but was not %s: %w", debugStr(contentPath.originalRequestedPath.String()), err) + webError(w, err, http.StatusInternalServerError) return } + contentPath.immutableRequestedPath = immutableContentPath } + contentPath.immutablePartiallyResolvedPath = contentPath.immutableRequestedPath - if err := i.handleGettingFirstBlock(r, begin, contentPath, resolvedPath); err != nil { - webRequestError(w, err) + // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified + ifNoneMatchResolvedPath, ok := i.handleIfNoneMatch(w, r, responseFormat, contentPath, logger) + if !ok { return } - if err := i.setCommonHeaders(w, r, contentPath); err != nil { - webRequestError(w, err) - return + // If we already did the path resolution no need to do it again + if ifNoneMatchResolvedPath != nil { + maybeResolvedImPath, err := NewImmutablePath(ifNoneMatchResolvedPath) + if err != nil { + webError(w, err, http.StatusInternalServerError) + return + } + contentPath.immutablePartiallyResolvedPath = maybeResolvedImPath } var success bool @@ -387,30 +407,21 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { // Support custom response formats passed via ?format or Accept HTTP header switch responseFormat { case "", "application/json", "application/cbor": - switch mc.Code(resolvedPath.Cid().Prefix().Codec) { - case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor: - logger.Debugw("serving codec", "path", contentPath) - success = i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) - default: - logger.Debugw("serving unixfs", "path", contentPath) - success = i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger) - } + success = i.serveDefaults(r.Context(), w, r, contentPath, begin, responseFormat, logger) case "application/vnd.ipld.raw": logger.Debugw("serving raw block", "path", contentPath) - success = i.serveRawBlock(r.Context(), w, r, resolvedPath, contentPath, begin) + success = i.serveRawBlock(r.Context(), w, r, contentPath, begin) case "application/vnd.ipld.car": logger.Debugw("serving car stream", "path", contentPath) carVersion := formatParams["version"] - success = i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin) + success = i.serveCAR(r.Context(), w, r, contentPath, carVersion, begin) case "application/x-tar": logger.Debugw("serving tar file", "path", contentPath) - success = i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + success = i.serveTAR(r.Context(), w, r, contentPath, begin, logger) case "application/vnd.ipld.dag-json", "application/vnd.ipld.dag-cbor": logger.Debugw("serving codec", "path", contentPath) - success = i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) + success = i.serveCodec(r.Context(), w, r, contentPath, begin, responseFormat) case "application/vnd.ipfs.ipns-record": - logger.Debugw("serving ipns record", "path", contentPath) - success = i.serveIpnsRecord(r.Context(), w, r, resolvedPath, contentPath, begin, logger) default: // catch-all for unsuported application/vnd.* err := fmt.Errorf("unsupported format %q", responseFormat) webError(w, err, http.StatusBadRequest) @@ -418,7 +429,7 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } if success { - i.getMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.getMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) } } @@ -428,12 +439,12 @@ func (i *handler) addUserHeaders(w http.ResponseWriter) { } } -func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, fileCid cid.Cid) (modtime time.Time) { +func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, fileCid cid.Cid) (modtime time.Time) { // Set Etag to based on CID (override whatever was set before) w.Header().Set("Etag", getEtag(r, fileCid)) // Set Cache-Control and Last-Modified based on contentPath properties - if contentPath.Mutable() { + if contentPath.originalRequestedPath.Mutable() { // mutable namespaces such as /ipns/ can't be cached forever /* For now we set Last-Modified to Now() to leverage caching heuristics built into modern browsers: @@ -489,7 +500,7 @@ func setContentDispositionHeader(w http.ResponseWriter, filename string, disposi } // Set X-Ipfs-Roots with logical CID array for efficient HTTP cache invalidation. -func (i *handler) buildIpfsRootsHeader(contentPath string, r *http.Request) (string, error) { +func (i *handler) setIpfsRootsHeader(w http.ResponseWriter, pathMetadata ContentPathMetadata) *ErrorResponse { /* These are logical roots where each CID represent one path segment and resolves to either a directory or the root block of a file. @@ -512,24 +523,16 @@ func (i *handler) buildIpfsRootsHeader(contentPath string, r *http.Request) (str Note that while the top one will change every time any article is changed, the last root (responsible for specific article) may not change at all. */ - var sp strings.Builder + var pathRoots []string - pathSegments := strings.Split(contentPath[6:], "/") - sp.WriteString(contentPath[:5]) // /ipfs or /ipns - for _, root := range pathSegments { - if root == "" { - continue - } - sp.WriteString("/") - sp.WriteString(root) - resolvedSubPath, err := i.api.ResolvePath(r.Context(), ipath.New(sp.String())) - if err != nil { - return "", err - } - pathRoots = append(pathRoots, resolvedSubPath.Cid().String()) + for _, c := range pathMetadata.PathSegmentRoots { + pathRoots = append(pathRoots, c.String()) } + pathRoots = append(pathRoots, pathMetadata.LastSegment.Cid().String()) rootCidList := strings.Join(pathRoots, ",") // convention from rfc2616#sec4.2 - return rootCidList, nil + + w.Header().Set("X-Ipfs-Roots", rootCidList) + return nil } func getFilename(contentPath ipath.Path) string { @@ -680,60 +683,90 @@ func debugStr(path string) string { return q } -// Resolve the provided contentPath including any special handling related to -// the requested responseFormat. Returned ok flag indicates if gateway handler -// should continue processing the request. -func (i *handler) handlePathResolution(w http.ResponseWriter, r *http.Request, responseFormat string, contentPath ipath.Path, logger *zap.SugaredLogger) (resolvedPath ipath.Resolved, newContentPath ipath.Path, ok bool) { - // Attempt to resolve the provided path. - resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath) +func (i *handler) handleIfNoneMatch(w http.ResponseWriter, r *http.Request, responseFormat string, contentPath contentPathRequest, logger *zap.SugaredLogger) (ipath.Resolved, bool) { + // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified + if inm := r.Header.Get("If-None-Match"); inm != "" { + pathMetadata, err := i.api.ResolvePath(r.Context(), contentPath.immutableRequestedPath) + if err != nil { + // Note: webError will replace http.StatusInternalServerError with a more appropriate error (e.g. StatusNotFound, StatusRequestTimeout, StatusServiceUnavailable, etc.) if necessary + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.originalRequestedPath.String()), err) + webError(w, err, http.StatusInternalServerError) + return nil, false + } - switch err { - case nil: - return resolvedPath, contentPath, true - case coreiface.ErrOffline: - err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) - webError(w, err, http.StatusServiceUnavailable) - return nil, nil, false - default: - // The path can't be resolved. - if isUnixfsResponseFormat(responseFormat) { - // If we have origin isolation (subdomain gw, DNSLink website), - // and response type is UnixFS (default for website hosting) - // check for presence of _redirects file and apply rules defined there. - // See: https://github.com/ipfs/specs/pull/290 - if hasOriginIsolation(r) { - resolvedPath, newContentPath, ok, hadMatchingRule := i.serveRedirectsIfPresent(w, r, resolvedPath, contentPath, logger) - if hadMatchingRule { - logger.Debugw("applied a rule from _redirects file") - return resolvedPath, newContentPath, ok - } - } + resolvedPath := pathMetadata.LastSegment + pathCid := resolvedPath.Cid() + // need to check against both File and Dir Etag variants + // because this inexpensive check happens before we do any I/O + cidEtag := getEtag(r, pathCid) + dirEtag := getDirListingEtag(pathCid) + if etagMatch(inm, cidEtag, dirEtag) { + // Finish early if client already has a matching Etag + w.WriteHeader(http.StatusNotModified) + return nil, false + } - // if Accept is text/html, see if ipfs-404.html is present - // This logic isn't documented and will likely be removed at some point. - // Any 404 logic in _redirects above will have already run by this time, so it's really an extra fall back - if i.serveLegacy404IfPresent(w, r, contentPath) { - logger.Debugw("served legacy 404") - return nil, nil, false - } + return resolvedPath, true + } + return nil, true +} + +func (i *handler) handleNonUnixFSRequestErrors(w http.ResponseWriter, contentPath ipath.Path, err error) bool { + if err == nil { + return true + } + // Note: webError will replace http.StatusInternalServerError with a more appropriate error (e.g. StatusNotFound, StatusRequestTimeout, StatusServiceUnavailable, etc.) if necessary + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + webError(w, err, http.StatusInternalServerError) + return false +} + +func (i *handler) handleUnixFSRequestErrors(w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, err error, logger *zap.SugaredLogger) (ImmutablePath, bool) { + if err == nil { + return contentPath.immutablePartiallyResolvedPath, true + } + + if errors.Is(err, ErrServiceUnavailable) { + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.originalRequestedPath.String()), err) + webError(w, err, http.StatusServiceUnavailable) + return ImmutablePath{}, false + } + + // If we have origin isolation (subdomain gw, DNSLink website), + // and response type is UnixFS (default for website hosting) + // we can leverage the presence of an _redirects file and apply rules defined there. + // See: https://github.com/ipfs/specs/pull/290 + if hasOriginIsolation(r) { + newContentPath, ok, hadMatchingRule := i.serveRedirectsIfPresent(w, r, contentPath, logger) + if hadMatchingRule { + logger.Debugw("applied a rule from _redirects file") + return newContentPath, ok } + } - err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) - webError(w, err, http.StatusInternalServerError) - return nil, nil, false + // if Accept is text/html, see if ipfs-404.html is present + // This logic isn't documented and will likely be removed at some point. + // Any 404 logic in _redirects above will have already run by this time, so it's really an extra fall back + if i.serveLegacy404IfPresent(w, r, contentPath.immutableRequestedPath) { + logger.Debugw("served legacy 404") + return ImmutablePath{}, false } + + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.originalRequestedPath.String()), err) + webError(w, err, http.StatusInternalServerError) + return ImmutablePath{}, false } // Detect 'Cache-Control: only-if-cached' in request and return data if it is already in the local datastore. // https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#cache-control-request-header -func (i *handler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) (requestHandled bool) { +func (i *handler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, logger *zap.SugaredLogger) (requestHandled bool) { if r.Header.Get("Cache-Control") == "only-if-cached" { - if !i.api.IsCached(r.Context(), contentPath) { + if !i.api.IsCached(r.Context(), contentPath.originalRequestedPath) { if r.Method == http.MethodHead { w.WriteHeader(http.StatusPreconditionFailed) return true } - errMsg := fmt.Sprintf("%q not in local datastore", contentPath.String()) + errMsg := fmt.Sprintf("%q not in local datastore", contentPath.originalRequestedPath.String()) http.Error(w, errMsg, http.StatusPreconditionFailed) return true } @@ -802,9 +835,9 @@ func handleServiceWorkerRegistration(r *http.Request) (err *ErrorResponse) { // 'intended' path is valid. This is in case gremlins were tickled // wrong way and user ended up at /ipfs/ipfs/{cid} or /ipfs/ipns/{id} // like in bafybeien3m7mdn6imm425vc2s22erzyhbvk5n3ofzgikkhmdkh5cuqbpbq :^)) -func handleSuperfluousNamespace(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) (requestHandled bool) { +func handleSuperfluousNamespace(w http.ResponseWriter, r *http.Request, contentPath contentPathRequest) (requestHandled bool) { // If the path is valid, there's nothing to do - if pathErr := contentPath.IsValid(); pathErr == nil { + if pathErr := contentPath.originalRequestedPath.IsValid(); pathErr == nil { return false } @@ -842,35 +875,6 @@ func handleSuperfluousNamespace(w http.ResponseWriter, r *http.Request, contentP return true } -func (i *handler) handleGettingFirstBlock(r *http.Request, begin time.Time, contentPath ipath.Path, resolvedPath ipath.Resolved) *ErrorResponse { - // Update the global metric of the time it takes to read the final root block of the requested resource - // NOTE: for legacy reasons this happens before we go into content-type specific code paths - _, err := i.api.GetBlock(r.Context(), resolvedPath.Cid()) - if err != nil { - err = fmt.Errorf("could not get block %s: %w", resolvedPath.Cid().String(), err) - return NewErrorResponse(err, http.StatusInternalServerError) - } - ns := contentPath.Namespace() - timeToGetFirstContentBlock := time.Since(begin).Seconds() - i.unixfsGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock) // deprecated, use firstContentBlockGetMetric instead - i.firstContentBlockGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock) - return nil -} - -func (i *handler) setCommonHeaders(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) *ErrorResponse { - i.addUserHeaders(w) // ok, _now_ write user's headers. - w.Header().Set("X-Ipfs-Path", contentPath.String()) - - if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil { - w.Header().Set("X-Ipfs-Roots", rootCids) - } else { // this should never happen, as we resolved the contentPath already - err = fmt.Errorf("error while resolving X-Ipfs-Roots: %w", err) - return NewErrorResponse(err, http.StatusInternalServerError) - } - - return nil -} - // spanTrace starts a new span using the standard IPFS tracing conventions. func spanTrace(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { return otel.Tracer("go-libipfs").Start(ctx, fmt.Sprintf("%s.%s", " Gateway", spanName), opts...) diff --git a/gateway/handler_block.go b/gateway/handler_block.go index a4f00ff9d..bc4c9a216 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -1,30 +1,31 @@ package gateway import ( - "bytes" "context" - "fmt" "net/http" "time" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) // serveRawBlock returns bytes behind a raw block -func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time) bool { - ctx, span := spanTrace(ctx, "ServeRawBlock", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, begin time.Time) bool { + ctx, span := spanTrace(ctx, "ServeRawBlock", trace.WithAttributes(attribute.String("path", contentPath.immutablePartiallyResolvedPath.String()))) defer span.End() - blockCid := resolvedPath.Cid() - block, err := i.api.GetBlock(ctx, blockCid) - if err != nil { - err = fmt.Errorf("error getting block %s: %w", blockCid.String(), err) - webError(w, err, http.StatusInternalServerError) + pathMetadata, data, err := i.api.GetBlock(ctx, contentPath.immutablePartiallyResolvedPath) + if !i.handleNonUnixFSRequestErrors(w, contentPath.immutablePartiallyResolvedPath, err) { return false } - content := bytes.NewReader(block.RawData()) + defer data.Close() + + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + + blockCid := pathMetadata.LastSegment.Cid() // Set Content-Disposition var name string @@ -42,11 +43,11 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h // ServeContent will take care of // If-None-Match+Etag, Content-Length and range requests - _, dataSent, _ := ServeContent(w, r, name, modtime, content) + _, dataSent, _ := ServeContent(w, r, name, modtime, data) if dataSent { // Update metrics - i.rawBlockGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.rawBlockGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) } return dataSent diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 0ccb60200..e323dfccd 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -3,21 +3,18 @@ package gateway import ( "context" "fmt" + "io" "net/http" "time" - cid "github.com/ipfs/go-cid" - blocks "github.com/ipfs/go-libipfs/blocks" - ipath "github.com/ipfs/interface-go-ipfs-core/path" - gocar "github.com/ipld/go-car" - selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/multierr" ) // serveCAR returns a CAR stream for specific DAG+selector -func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, carVersion string, begin time.Time) bool { - ctx, span := spanTrace(ctx, "ServeCAR", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, carVersion string, begin time.Time) bool { + ctx, span := spanTrace(ctx, "ServeCAR", trace.WithAttributes(attribute.String("path", contentPath.immutablePartiallyResolvedPath.String()))) defer span.End() ctx, cancel := context.WithCancel(ctx) @@ -31,7 +28,19 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R webError(w, err, http.StatusBadRequest) return false } - rootCid := resolvedPath.Cid() + + pathMetadata, carFile, errCh, err := i.api.GetCAR(ctx, contentPath.immutablePartiallyResolvedPath) + if !i.handleNonUnixFSRequestErrors(w, contentPath.originalRequestedPath, err) { + return false + } + defer carFile.Close() + + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + + rootCid := pathMetadata.LastSegment.Cid() // Set Content-Disposition var name string @@ -67,33 +76,18 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R w.Header().Set("Content-Type", "application/vnd.ipld.car; version=1") w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) - // Same go-car settings as dag.export command - store := dagStore{api: i.api, ctx: ctx} - - // TODO: support selectors passed as request param: https://github.com/ipfs/kubo/issues/8769 - dag := gocar.Dag{Root: rootCid, Selector: selectorparse.CommonSelector_ExploreAllRecursively} - car := gocar.NewSelectiveCar(ctx, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce()) - - if err := car.Write(w); err != nil { + _, copyErr := io.Copy(w, carFile) + carErr := <-errCh + if copyErr != nil || carErr != nil { // We return error as a trailer, however it is not something browsers can access // (https://github.com/mdn/browser-compat-data/issues/14703) // Due to this, we suggest client always verify that // the received CAR stream response is matching requested DAG selector - w.Header().Set("X-Stream-Error", err.Error()) + w.Header().Set("X-Stream-Error", multierr.Combine(err, copyErr).Error()) return false } // Update metrics - i.carStreamGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.carStreamGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) return true } - -// FIXME(@Jorropo): https://github.com/ipld/go-car/issues/315 -type dagStore struct { - api API - ctx context.Context -} - -func (ds dagStore) Get(_ context.Context, c cid.Cid) (blocks.Block, error) { - return ds.api.GetBlock(ds.ctx, c) -} diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index ed02ab107..2c31f802e 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -4,13 +4,13 @@ import ( "bytes" "context" "fmt" + "io" "net/http" "strings" "time" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/ipfs/go-libipfs/gateway/assets" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipld/go-ipld-prime/multicodec" "github.com/ipld/go-ipld-prime/node/basicnode" mc "github.com/multiformats/go-multicodec" @@ -50,19 +50,39 @@ var contentTypeToExtension = map[string]string{ "application/vnd.ipld.dag-cbor": ".cbor", } -func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, requestedContentType string) bool { - ctx, span := spanTrace(ctx, "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) +func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, begin time.Time, requestedContentType string) bool { + ctx, span := spanTrace(ctx, "ServeCodec", trace.WithAttributes(attribute.String("path", contentPath.immutablePartiallyResolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) defer span.End() - cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) + pathMetadata, data, err := i.api.GetBlock(ctx, contentPath.immutablePartiallyResolvedPath) + if !i.handleNonUnixFSRequestErrors(w, contentPath.originalRequestedPath, err) { + return false + } + defer data.Close() + + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + + contentPath.finalResolvedPath = pathMetadata.LastSegment + return i.renderCodec(ctx, w, r, contentPath, data, begin, requestedContentType) +} + +func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, blockData io.ReadSeekCloser, begin time.Time, requestedContentType string) bool { + ctx, span := spanTrace(ctx, "RenderCodec", trace.WithAttributes(attribute.String("path", contentPath.finalResolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) + defer span.End() + + blockCid := contentPath.finalResolvedPath.Cid() + cidCodec := mc.Code(blockCid.Prefix().Codec) responseContentType := requestedContentType // If the resolved path still has some remainder, return error for now. // TODO: handle this when we have IPLD Patch (https://ipld.io/specs/patch/) via HTTP PUT // TODO: (depends on https://github.com/ipfs/kubo/issues/4801 and https://github.com/ipfs/kubo/issues/4782) - if resolvedPath.Remainder() != "" { - path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder()) - err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", resolvedPath.Remainder(), resolvedPath.String(), path) + if contentPath.finalResolvedPath.Remainder() != "" { + path := strings.TrimSuffix(contentPath.finalResolvedPath.String(), contentPath.finalResolvedPath.Remainder()) + err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", contentPath.finalResolvedPath.Remainder(), contentPath.finalResolvedPath.String(), path) webError(w, err, http.StatusNotImplemented) return false } @@ -80,8 +100,8 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http } // Set HTTP headers (for caching etc) - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) - name := setCodecContentDisposition(w, r, resolvedPath, responseContentType) + modtime := addCacheControlHeaders(w, r, contentPath, contentPath.finalResolvedPath.Cid()) + name := setCodecContentDisposition(w, r, contentPath, responseContentType) w.Header().Set("Content-Type", responseContentType) w.Header().Set("X-Content-Type-Options", "nosniff") @@ -93,11 +113,11 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http download := r.URL.Query().Get("download") == "true" if isDAG && acceptsHTML && !download { - return i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath) + return i.serveCodecHTML(ctx, w, r, contentPath) } else { // This covers CIDs with codec 'json' and 'cbor' as those do not have // an explicit requested content type. - return i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime, begin) + return i.serveCodecRaw(ctx, w, r, blockData, contentPath, name, modtime, begin) } } @@ -107,7 +127,7 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http if ok { for _, skipCodec := range skipCodecs { if skipCodec == cidCodec { - return i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime, begin) + return i.serveCodecRaw(ctx, w, r, blockData, contentPath, name, modtime, begin) } } } @@ -123,10 +143,10 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http } // This handles DAG-* conversions and validations. - return i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime, begin) + return i.serveCodecConverted(ctx, w, r, blockCid, blockData, contentPath, toCodec, modtime, begin) } -func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { +func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest) bool { // A HTML directory index will be presented, be sure to set the correct // type instead of relying on autodetection (which may fail). w.Header().Set("Content-Type", "text/html") @@ -135,7 +155,7 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * w.Header().Del("Content-Disposition") // Generated index requires custom Etag (output may change between Kubo versions) - dagEtag := getDagIndexEtag(resolvedPath.Cid()) + dagEtag := getDagIndexEtag(contentPath.finalResolvedPath.Cid()) w.Header().Set("Etag", dagEtag) // Remove Cache-Control for now to match UnixFS dir-index-html responses @@ -143,10 +163,10 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * // TODO: if we ever change behavior for UnixFS dir listings, same changes should be applied here w.Header().Del("Cache-Control") - cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) + cidCodec := mc.Code(contentPath.finalResolvedPath.Cid().Prefix().Codec) if err := assets.DagTemplate.Execute(w, assets.DagTemplateData{ - Path: contentPath.String(), - CID: resolvedPath.Cid().String(), + Path: contentPath.originalRequestedPath.String(), + CID: contentPath.finalResolvedPath.Cid().String(), CodecName: cidCodec.String(), CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), }); err != nil { @@ -159,38 +179,21 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * } // serveCodecRaw returns the raw block without any conversion -func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, name string, modtime, begin time.Time) bool { - blockCid := resolvedPath.Cid() - block, err := i.api.GetBlock(ctx, blockCid) - if err != nil { - err = fmt.Errorf("error getting block %s: %w", blockCid.String(), err) - webError(w, err, http.StatusInternalServerError) - return false - } - content := bytes.NewReader(block.RawData()) - +func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockData io.ReadSeekCloser, contentPath contentPathRequest, name string, modtime, begin time.Time) bool { // ServeContent will take care of // If-None-Match+Etag, Content-Length and range requests - _, dataSent, _ := ServeContent(w, r, name, modtime, content) + _, dataSent, _ := ServeContent(w, r, name, modtime, blockData) if dataSent { // Update metrics - i.jsoncborDocumentGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.jsoncborDocumentGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) } return dataSent } // serveCodecConverted returns payload converted to codec specified in toCodec -func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec mc.Code, modtime, begin time.Time) bool { - blockCid := resolvedPath.Cid() - block, err := i.api.GetBlock(ctx, blockCid) - if err != nil { - err = fmt.Errorf("error getting block %s: %w", blockCid.String(), err) - webError(w, err, http.StatusInternalServerError) - return false - } - +func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadSeekCloser, contentPath contentPathRequest, toCodec mc.Code, modtime, begin time.Time) bool { codec := blockCid.Prefix().Codec decoder, err := multicodec.LookupDecoder(codec) if err != nil { @@ -199,7 +202,7 @@ func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter } node := basicnode.Prototype.Any.NewBuilder() - err = decoder(node, bytes.NewReader(block.RawData())) + err = decoder(node, blockData) if err != nil { webError(w, err, http.StatusInternalServerError) return false @@ -228,14 +231,14 @@ func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter _, err = w.Write(buf.Bytes()) if err == nil { // Update metrics - i.jsoncborDocumentGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.jsoncborDocumentGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) return true } return false } -func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentType string) string { +func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, contentType string) string { var dispType, name string ext, ok := contentTypeToExtension[contentType] @@ -247,7 +250,7 @@ func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolved if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" { name = urlFilename } else { - name = resolvedPath.Cid().String() + ext + name = contentPath.finalResolvedPath.Cid().String() + ext } // JSON should be inlined, but ?download=true should still override diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go new file mode 100644 index 000000000..89996fff4 --- /dev/null +++ b/gateway/handler_defaults.go @@ -0,0 +1,203 @@ +package gateway + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/textproto" + "strconv" + "strings" + "time" + + "github.com/ipfs/go-libipfs/files" + mc "github.com/multiformats/go-multicodec" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, begin time.Time, requestedContentType string, logger *zap.SugaredLogger) bool { + ctx, span := spanTrace(ctx, "ServeDefaults", trace.WithAttributes(attribute.String("path", contentPath.originalRequestedPath.String()))) + defer span.End() + + var ( + pathMetadata ContentPathMetadata + bytesResponse files.File + isDirectoryHeadRequest bool + dirListing *directoryResponse + err error + ) + + switch r.Method { + case http.MethodHead: + var data files.Node + pathMetadata, data, err = i.api.Head(ctx, contentPath.immutablePartiallyResolvedPath) + if !i.handleNonUnixFSRequestErrors(w, contentPath.originalRequestedPath, err) { // TODO: even though this might be UnixFS there shouldn't be anything special for HEAD requests + return false + } + defer data.Close() + if _, ok := data.(files.Directory); ok { + isDirectoryHeadRequest = true + } else if f, ok := data.(files.File); ok { + bytesResponse = f + } else { + webError(w, fmt.Errorf("unsupported response type"), http.StatusInternalServerError) + return false + } + case http.MethodGet: + rangeHeader := r.Header.Get("Range") + if rangeHeader == "" { + var getResp *GetResponse + pathMetadata, getResp, err = i.api.Get(ctx, contentPath.immutablePartiallyResolvedPath) + if err != nil { + if isUnixfsResponseFormat(requestedContentType) { + forwardedPath, continueProcessing := i.handleUnixFSRequestErrors(w, r, contentPath, err, logger) + if !continueProcessing { + return false + } + pathMetadata, getResp, err = i.api.Get(ctx, forwardedPath) + if err != nil { + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.originalRequestedPath.String()), err) + webError(w, err, http.StatusInternalServerError) + } + } else { + if !i.handleNonUnixFSRequestErrors(w, contentPath.originalRequestedPath, err) { + return false + } + } + } + if getResp.bytes != nil { + bytesResponse = getResp.bytes + defer bytesResponse.Close() + } else { + dirListing = getResp.directory + } + } else { + // TODO: Add tests for range parsing + var ranges []GetRange + ranges, err = parseRange(rangeHeader) + if err != nil { + webError(w, fmt.Errorf("invalid range request: %w", err), http.StatusBadRequest) + return false + } + pathMetadata, bytesResponse, err = i.api.GetRange(ctx, contentPath.immutablePartiallyResolvedPath, ranges...) + if err != nil { + if isUnixfsResponseFormat(requestedContentType) { + forwardedPath, continueProcessing := i.handleUnixFSRequestErrors(w, r, contentPath, err, logger) + if !continueProcessing { + return false + } + pathMetadata, bytesResponse, err = i.api.GetRange(ctx, forwardedPath, ranges...) + if err != nil { + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.originalRequestedPath.String()), err) + webError(w, err, http.StatusInternalServerError) + } + } else { + if !i.handleNonUnixFSRequestErrors(w, contentPath.originalRequestedPath, err) { + return false + } + } + } + defer bytesResponse.Close() + } + default: + // This shouldn't be possible to reach which is why it is a 500 rather than 4XX error + webError(w, fmt.Errorf("invalid method: cannot use this HTTP method with the given request"), http.StatusInternalServerError) + return false + } + + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + + contentPath.finalResolvedPath = pathMetadata.LastSegment + switch mc.Code(contentPath.finalResolvedPath.Cid().Prefix().Codec) { + case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor: + if bytesResponse == nil { // This should never happen + webError(w, fmt.Errorf("decoding error: data not a usable as a file"), http.StatusInternalServerError) + return false + } + logger.Debugw("serving codec", "path", contentPath.originalRequestedPath) + return i.renderCodec(r.Context(), w, r, contentPath, bytesResponse, begin, requestedContentType) + default: + logger.Debugw("serving unixfs", "path", contentPath) + ctx, span := spanTrace(ctx, "ServeUnixFS", trace.WithAttributes(attribute.String("path", contentPath.finalResolvedPath.String()))) + defer span.End() + + // Handling Unixfs file + if bytesResponse != nil { + logger.Debugw("serving unixfs file", "path", contentPath.originalRequestedPath) + return i.serveFile(ctx, w, r, contentPath, bytesResponse, pathMetadata.ContentType, begin) + } + + // Handling Unixfs directory + if !isDirectoryHeadRequest && dirListing == nil { + webError(w, fmt.Errorf("unsupported UnixFS type"), http.StatusInternalServerError) + return false + } + + logger.Debugw("serving unixfs directory", "path", contentPath.originalRequestedPath) + return i.serveDirectory(ctx, w, r, contentPath, isDirectoryHeadRequest, dirListing, begin, logger) + } +} + +// parseRange parses a Range header string as per RFC 7233. +func parseRange(s string) ([]GetRange, error) { + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, errors.New("invalid range") + } + var ranges []GetRange + for _, ra := range strings.Split(s[len(b):], ",") { + ra = textproto.TrimString(ra) + if ra == "" { + continue + } + start, end, ok := strings.Cut(ra, "-") + if !ok { + return nil, errors.New("invalid range") + } + start, end = textproto.TrimString(start), textproto.TrimString(end) + var r GetRange + if start == "" { + r.From = 0 + // If no start is specified, end specifies the + // range start relative to the end of the file, + // and we are dealing with + // which has to be a non-negative integer as per + // RFC 7233 Section 2.1 "Byte-Ranges". + if end == "" || end[0] == '-' { + return nil, errors.New("invalid range") + } + i, err := strconv.ParseInt(end, 10, 64) + if i < 0 || err != nil { + return nil, errors.New("invalid range") + } + r.To = &i + } else { + i, err := strconv.ParseUint(start, 10, 64) + if err != nil { + return nil, errors.New("invalid range") + } + r.From = i + if end == "" { + // If no end is specified, range extends to end of the file. + r.To = nil + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || i < 0 || r.From > uint64(i) { + return nil, errors.New("invalid range") + } + r.To = &i + } + } + ranges = append(ranges, r) + } + return ranges, nil +} diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go index e9548f777..5105fd8a5 100644 --- a/gateway/handler_ipns_record.go +++ b/gateway/handler_ipns_record.go @@ -5,29 +5,30 @@ import ( "errors" "fmt" "net/http" + "strconv" "strings" "time" + "github.com/cespare/xxhash" "github.com/gogo/protobuf/proto" "github.com/ipfs/go-cid" ipns_pb "github.com/ipfs/go-ipns/pb" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) -func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "ServeIPNSRecord", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, begin time.Time, logger *zap.SugaredLogger) bool { + ctx, span := spanTrace(ctx, "ServeIPNSRecord", trace.WithAttributes(attribute.String("path", contentPath.originalRequestedPath.String()))) defer span.End() - if contentPath.Namespace() != "ipns" { - err := fmt.Errorf("%s is not an IPNS link", contentPath.String()) + if contentPath.originalRequestedPath.Namespace() != "ipns" { + err := fmt.Errorf("%s is not an IPNS link", contentPath.originalRequestedPath.String()) webError(w, err, http.StatusBadRequest) return false } - key := contentPath.String() + key := contentPath.originalRequestedPath.String() key = strings.TrimSuffix(key, "/") key = strings.TrimPrefix(key, "/ipns/") if strings.Count(key, "/") != 0 { @@ -59,7 +60,8 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r // TTL is not present, we use the Last-Modified tag. We are tracking IPNS // caching on: https://github.com/ipfs/kubo/issues/1818. // TODO: use addCacheControlHeaders once #1818 is fixed. - w.Header().Set("Etag", getEtag(r, resolvedPath.Cid())) + recordEtag := strconv.FormatUint(xxhash.Sum64(rawRecord), 32) + w.Header().Set("Etag", recordEtag) if record.Ttl != nil { seconds := int(time.Duration(*record.Ttl).Seconds()) w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", seconds)) @@ -82,7 +84,7 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r _, err = w.Write(rawRecord) if err == nil { // Update metrics - i.ipnsRecordGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.ipnsRecordGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) return true } diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index 569c031d4..72e6fb26f 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -3,12 +3,10 @@ package gateway import ( "context" "fmt" - "html" "net/http" "time" "github.com/ipfs/go-libipfs/files" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -16,23 +14,25 @@ import ( var unixEpochTime = time.Unix(0, 0) -func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "ServeTAR", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, begin time.Time, logger *zap.SugaredLogger) bool { + ctx, span := spanTrace(ctx, "ServeTAR", trace.WithAttributes(attribute.String("path", contentPath.immutablePartiallyResolvedPath.String()))) defer span.End() ctx, cancel := context.WithCancel(ctx) defer cancel() - // Get Unixfs file - file, err := i.api.GetUnixFsNode(ctx, resolvedPath) - if err != nil { - err = fmt.Errorf("error getting UnixFS node for %s: %w", html.EscapeString(contentPath.String()), err) - webError(w, err, http.StatusInternalServerError) + // Get Unixfs file (or directory) + pathMetadata, file, err := i.api.GetAll(ctx, contentPath.immutablePartiallyResolvedPath) + if !i.handleNonUnixFSRequestErrors(w, contentPath.originalRequestedPath, err) { return false } defer file.Close() - rootCid := resolvedPath.Cid() + if err := i.setIpfsRootsHeader(w, pathMetadata); err != nil { + webRequestError(w, err) + return false + } + rootCid := pathMetadata.LastSegment.Cid() // Set Cache-Control and read optional Last-Modified time modtime := addCacheControlHeaders(w, r, contentPath, rootCid) @@ -92,6 +92,6 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R } // Update metrics - i.tarStreamGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.tarStreamGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) return true } diff --git a/gateway/handler_test.go b/gateway/handler_test.go index c3dcf9606..9bdc1bfd8 100644 --- a/gateway/handler_test.go +++ b/gateway/handler_test.go @@ -3,6 +3,7 @@ package gateway import ( "context" "fmt" + "io" "net/http" "testing" @@ -10,10 +11,8 @@ import ( cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/files" "github.com/ipfs/go-path/resolver" - iface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" "github.com/stretchr/testify/assert" ) @@ -45,16 +44,32 @@ type errorMockAPI struct { err error } -func (api *errorMockAPI) GetUnixFsNode(context.Context, ipath.Resolved) (files.Node, error) { - return nil, api.err +func (api *errorMockAPI) Get(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *GetResponse, error) { + return ContentPathMetadata{}, nil, api.err } -func (api *errorMockAPI) LsUnixFsDir(ctx context.Context, p ipath.Resolved) (<-chan iface.DirEntry, error) { - return nil, api.err +func (api *errorMockAPI) GetRange(ctx context.Context, path ImmutablePath, getRange ...GetRange) (ContentPathMetadata, files.File, error) { + return ContentPathMetadata{}, nil, api.err } -func (api *errorMockAPI) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return nil, api.err +func (api *errorMockAPI) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + return ContentPathMetadata{}, nil, api.err +} + +func (api *errorMockAPI) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { + return ContentPathMetadata{}, nil, api.err +} + +func (api *errorMockAPI) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + return ContentPathMetadata{}, nil, api.err +} + +func (api *errorMockAPI) GetCAR(ctx context.Context, path ImmutablePath) (ContentPathMetadata, io.ReadCloser, <-chan error, error) { + return ContentPathMetadata{}, nil, nil, api.err +} + +func (api *errorMockAPI) ResolveMutable(ctx context.Context, path ipath.Path) (ImmutablePath, error) { + return ImmutablePath{}, api.err } func (api *errorMockAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { @@ -69,8 +84,8 @@ func (api *errorMockAPI) IsCached(ctx context.Context, p ipath.Path) bool { return false } -func (api *errorMockAPI) ResolvePath(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { - return nil, api.err +func (api *errorMockAPI) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { + return ContentPathMetadata{}, api.err } func TestGatewayInternalServerErrorInvalidPath(t *testing.T) { diff --git a/gateway/handler_unixfs.go b/gateway/handler_unixfs.go deleted file mode 100644 index 28a0677d3..000000000 --- a/gateway/handler_unixfs.go +++ /dev/null @@ -1,44 +0,0 @@ -package gateway - -import ( - "context" - "fmt" - "net/http" - "time" - - "github.com/ipfs/go-libipfs/files" - ipath "github.com/ipfs/interface-go-ipfs-core/path" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" -) - -func (i *handler) serveUnixFS(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) - defer span.End() - - // Handling UnixFS - dr, err := i.api.GetUnixFsNode(ctx, resolvedPath) - if err != nil { - err = fmt.Errorf("error while getting UnixFS node: %w", err) - webError(w, err, http.StatusInternalServerError) - return false - } - defer dr.Close() - - // Handling Unixfs file - if f, ok := dr.(files.File); ok { - logger.Debugw("serving unixfs file", "path", contentPath) - return i.serveFile(ctx, w, r, resolvedPath, contentPath, f, begin) - } - - // Handling Unixfs directory - dir, ok := dr.(files.Directory) - if !ok { - webError(w, fmt.Errorf("unsupported UnixFS type"), http.StatusInternalServerError) - return false - } - - logger.Debugw("serving unixfs directory", "path", contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger) -} diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index b9658e4e5..0633a3a01 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -8,10 +8,12 @@ import ( "strconv" "strings" - redirects "github.com/ipfs/go-ipfs-redirects-file" + "go.uber.org/zap" + "github.com/ipfs/go-libipfs/files" ipath "github.com/ipfs/interface-go-ipfs-core/path" - "go.uber.org/zap" + + redirects "github.com/ipfs/go-ipfs-redirects-file" ) // Resolving a UnixFS path involves determining if the provided `path.Path` exists and returning the `path.Resolved` @@ -36,46 +38,58 @@ import ( // // Note that for security reasons, redirect rules are only processed when the request has origin isolation. // See https://github.com/ipfs/specs/pull/290 for more information. -func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, logger *zap.SugaredLogger) (newResolvedPath ipath.Resolved, newContentPath ipath.Path, continueProcessing bool, hadMatchingRule bool) { - redirectsFile := i.getRedirectsFile(r, contentPath, logger) - if redirectsFile != nil { - redirectRules, err := i.getRedirectRules(r, redirectsFile) - if err != nil { - webError(w, err, http.StatusInternalServerError) - return nil, nil, false, true - } +func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, logger *zap.SugaredLogger) (newContentPath ImmutablePath, continueProcessing bool, hadMatchingRule bool) { + // contentPath is the full ipfs path to the requested resource, + // regardless of whether path or subdomain resolution is used. + rootPath := getRootPath(contentPath.immutableRequestedPath) + redirectsPath := ipath.Join(rootPath, "_redirects") + imRedirectsPath, err := NewImmutablePath(redirectsPath) + if err != nil { + err = fmt.Errorf("trouble processing _redirects path %q: %w", redirectsPath, err) + webError(w, err, http.StatusInternalServerError) + return ImmutablePath{}, false, true + } + foundRedirect, redirectRules, err := i.getRedirectRules(r, imRedirectsPath) + if err != nil { + err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsPath, err) + webError(w, err, http.StatusInternalServerError) + return ImmutablePath{}, false, true + } + + if foundRedirect { redirected, newPath, err := i.handleRedirectsFileRules(w, r, contentPath, redirectRules) if err != nil { - err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsFile.String(), err) + err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsPath, err) webError(w, err, http.StatusInternalServerError) - return nil, nil, false, true + return ImmutablePath{}, false, true } if redirected { - return nil, nil, false, true + return ImmutablePath{}, false, true } // 200 is treated as a rewrite, so update the path and continue if newPath != "" { // Reassign contentPath and resolvedPath since the URL was rewritten - contentPath = ipath.New(newPath) - resolvedPath, err = i.api.ResolvePath(r.Context(), contentPath) + p := ipath.New(newPath) + imPath, err := NewImmutablePath(p) if err != nil { + err = fmt.Errorf("could not use _redirects file to %q: %w", p, err) webError(w, err, http.StatusInternalServerError) - return nil, nil, false, true + return ImmutablePath{}, false, true } - - return resolvedPath, contentPath, true, true + return imPath, true, true } } + // No matching rule, paths remain the same, continue regular processing - return resolvedPath, contentPath, true, false + return contentPath.immutablePartiallyResolvedPath, true, false } -func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, redirectRules []redirects.Rule) (redirected bool, newContentPath string, err error) { +func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Request, cPath contentPathRequest, redirectRules []redirects.Rule) (redirected bool, newContentPath string, err error) { // Attempt to match a rule to the URL path, and perform the corresponding redirect or rewrite - pathParts := strings.Split(contentPath.String(), "/") + pathParts := strings.Split(cPath.immutableRequestedPath.String(), "/") if len(pathParts) > 3 { // All paths should start with /ipfs/cid/, so get the path after that urlPath := "/" + strings.Join(pathParts[3:], "/") @@ -101,8 +115,24 @@ func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Reques // Or 4xx if rule.Status == 404 || rule.Status == 410 || rule.Status == 451 { toPath := rootPath + rule.To - content4xxPath := ipath.New(toPath) - err := i.serve4xx(w, r, content4xxPath, rule.Status) + imContent4xxPath, err := NewImmutablePath(ipath.New(toPath)) + if err != nil { + return true, toPath, err + } + + // While we have the immutable path which is enough to fetch the data we need to track mutability for + // headers. + contentPathParts := strings.Split(cPath.originalRequestedPath.String(), "/") + if len(contentPathParts) <= 3 { + // Match behavior as with the immutable path + return false, "", nil + } + // All paths should start with /ip(f|n)s//, so get the path after that + contentRootPath := strings.Join(contentPathParts[:3], "/") + content4xxPath := newPath(ipath.New(contentRootPath + rule.To)) + content4xxPath.immutableRequestedPath = imContent4xxPath + content4xxPath.immutablePartiallyResolvedPath = imContent4xxPath + err = i.serve4xx(w, r, content4xxPath, rule.Status) return true, toPath, err } @@ -118,44 +148,33 @@ func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Reques return false, "", nil } -func (i *handler) getRedirectRules(r *http.Request, redirectsFilePath ipath.Resolved) ([]redirects.Rule, error) { - // Convert the path into a file node - node, err := i.api.GetUnixFsNode(r.Context(), redirectsFilePath) +// getRedirectRules fetches the _redirects file corresponding to a given path and returns the rules +// Returns whether _redirects was found, the rules (if they exist) and if there was an error (other than a missing _redirects) +// If there is an error returns (false, nil, err) +func (i *handler) getRedirectRules(r *http.Request, redirectsPath ImmutablePath) (bool, []redirects.Rule, error) { + // Check for _redirects file. + // Any path resolution failures are ignored and we just assume there's no _redirects file. + // Note that ignoring these errors also ensures that the use of the empty CID (bafkqaaa) in tests doesn't fail. + _, redirectsFileGetResp, err := i.api.Get(r.Context(), redirectsPath) if err != nil { - return nil, fmt.Errorf("could not get _redirects: %w", err) + if isErrNotFound(err) { + return false, nil, nil + } + return false, nil, err } - defer node.Close() - // Convert the node into a file - f, ok := node.(files.File) - if !ok { - return nil, fmt.Errorf("could not parse _redirects: %w", err) + if redirectsFileGetResp.bytes == nil { + return false, nil, fmt.Errorf(" _redirects is not a file") } + f := redirectsFileGetResp.bytes + defer f.Close() // Parse redirect rules from file redirectRules, err := redirects.Parse(f) if err != nil { - return nil, fmt.Errorf("could not parse _redirects: %w", err) - } - - return redirectRules, nil -} - -// Returns a resolved path to the _redirects file located in the root CID path of the requested path -func (i *handler) getRedirectsFile(r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) ipath.Resolved { - // contentPath is the full ipfs path to the requested resource, - // regardless of whether path or subdomain resolution is used. - rootPath := getRootPath(contentPath) - - // Check for _redirects file. - // Any path resolution failures are ignored and we just assume there's no _redirects file. - // Note that ignoring these errors also ensures that the use of the empty CID (bafkqaaa) in tests doesn't fail. - path := ipath.Join(rootPath, "_redirects") - resolvedPath, err := i.api.ResolvePath(r.Context(), path) - if err != nil { - return nil + return false, nil, fmt.Errorf("could not parse _redirects: %w", err) } - return resolvedPath + return true, redirectRules, nil } // Returns the root CID Path for the given path @@ -164,24 +183,21 @@ func getRootPath(path ipath.Path) ipath.Path { return ipath.New(gopath.Join("/", path.Namespace(), parts[2])) } -func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPath ipath.Path, status int) error { - resolved4xxPath, err := i.api.ResolvePath(r.Context(), content4xxPath) - if err != nil { - return err - } - - node, err := i.api.GetUnixFsNode(r.Context(), resolved4xxPath) +func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPath contentPathRequest, status int) error { + pathMetadata, getresp, err := i.api.Get(r.Context(), content4xxPath.immutablePartiallyResolvedPath) if err != nil { return err } - defer node.Close() - f, ok := node.(files.File) - if !ok { + if getresp.bytes == nil { return fmt.Errorf("could not convert node for %d page to file", status) } + content4xxFile := getresp.bytes + defer content4xxFile.Close() + + content4xxCid := pathMetadata.LastSegment.Cid() - size, err := f.Size() + size, err := content4xxFile.Size() if err != nil { return fmt.Errorf("could not get size of %d page", status) } @@ -189,9 +205,9 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat log.Debugf("using _redirects: custom %d file at %q", status, content4xxPath) w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) - addCacheControlHeaders(w, r, content4xxPath, resolved4xxPath.Cid()) + addCacheControlHeaders(w, r, content4xxPath, content4xxCid) w.WriteHeader(status) - _, err = io.CopyN(w, f, size) + _, err = io.CopyN(w, content4xxFile, size) return err } @@ -214,43 +230,33 @@ func isUnixfsResponseFormat(responseFormat string) bool { // Deprecated: legacy ipfs-404.html files are superseded by _redirects file // This is provided only for backward-compatibility, until websites migrate // to 404s managed via _redirects file (https://github.com/ipfs/specs/pull/290) -func (i *handler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool { - resolved404Path, ctype, err := i.searchUpTreeFor404(r, contentPath) +func (i *handler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, imPath ImmutablePath) bool { + resolved404File, ctype, err := i.searchUpTreeFor404(r, imPath) if err != nil { return false } + defer resolved404File.Close() - dr, err := i.api.GetUnixFsNode(r.Context(), resolved404Path) + size, err := resolved404File.Size() if err != nil { return false } - defer dr.Close() - f, ok := dr.(files.File) - if !ok { - return false - } - - size, err := f.Size() - if err != nil { - return false - } - - log.Debugw("using pretty 404 file", "path", contentPath) + log.Debugw("using pretty 404 file", "path", imPath) w.Header().Set("Content-Type", ctype) w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) w.WriteHeader(http.StatusNotFound) - _, err = io.CopyN(w, f, size) + _, err = io.CopyN(w, resolved404File, size) return err == nil } -func (i *handler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (ipath.Resolved, string, error) { +func (i *handler) searchUpTreeFor404(r *http.Request, imPath ImmutablePath) (files.File, string, error) { filename404, ctype, err := preferred404Filename(r.Header.Values("Accept")) if err != nil { return nil, "", err } - pathComponents := strings.Split(contentPath.String(), "/") + pathComponents := strings.Split(imPath.String(), "/") for idx := len(pathComponents); idx >= 3; idx-- { pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...) @@ -258,11 +264,19 @@ func (i *handler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (i if parsed404Path.IsValid() != nil { break } - resolvedPath, err := i.api.ResolvePath(r.Context(), parsed404Path) + imparsed404Path, err := NewImmutablePath(parsed404Path) + if err != nil { + break + } + + _, getResp, err := i.api.Get(r.Context(), imparsed404Path) if err != nil { continue } - return resolvedPath, ctype, nil + if getResp.bytes == nil { + return nil, "", fmt.Errorf("found a pretty 404 but it was not a file") + } + return getResp.bytes, ctype, nil } return nil, "", fmt.Errorf("no pretty 404 in any parent folder") diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index c03a4b81b..df57c73e2 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -14,7 +14,6 @@ import ( "github.com/ipfs/go-libipfs/files" "github.com/ipfs/go-libipfs/gateway/assets" path "github.com/ipfs/go-path" - "github.com/ipfs/go-path/resolver" ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -24,8 +23,8 @@ import ( // serveDirectory returns the best representation of UnixFS directory // // It will return index.html if present, or generate directory listing otherwise. -func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, isHeadRequest bool, dirResp *directoryResponse, begin time.Time, logger *zap.SugaredLogger) bool { + ctx, span := spanTrace(ctx, "ServeDirectory", trace.WithAttributes(attribute.String("path", contentPath.finalResolvedPath.String()))) defer span.End() // HostnameOption might have constructed an IPNS/IPFS path using the Host header. @@ -60,32 +59,60 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } // Check if directory has index.html, if so, serveFile - idxPath := ipath.Join(contentPath, "index.html") - idxResolvedPath, err := i.api.ResolvePath(ctx, idxPath) - switch err.(type) { - case nil: - idx, err := i.api.GetUnixFsNode(ctx, idxResolvedPath) - if err != nil { - webError(w, err, http.StatusInternalServerError) - return false + idxPath := newPath(ipath.Join(contentPath.originalRequestedPath, "index.html")) + idxPathIm, err := NewImmutablePath(ipath.Join(contentPath.immutableRequestedPath, "index.html")) + if err != nil { + webError(w, err, http.StatusInternalServerError) + return false + } + idxPath.immutableRequestedPath = idxPathIm + idxPathPartiallyResolved, err := NewImmutablePath(ipath.Join(contentPath.finalResolvedPath, "index.html")) + if err != nil { + webError(w, err, http.StatusInternalServerError) + return false + } + idxPath.immutablePartiallyResolvedPath = idxPathPartiallyResolved + + // TODO: could/should this all be skipped to have HEAD requests just return html content type and save the complexity? If so can we skip the above code as well? + var idxFile files.File + var idxPathMd ContentPathMetadata + if isHeadRequest { + var idx files.Node + idxPathMd, idx, err = i.api.Head(ctx, idxPath.immutablePartiallyResolvedPath) + if err == nil { + f, ok := idx.(files.File) + if !ok { + webError(w, fmt.Errorf("%q could not be read: %w", idxPath.immutablePartiallyResolvedPath, files.ErrNotReader), http.StatusUnprocessableEntity) + return false + } + idxFile = f } - - f, ok := idx.(files.File) - if !ok { - webError(w, files.ErrNotReader, http.StatusInternalServerError) - return false + } else { + var getResp *GetResponse + idxPathMd, getResp, err = i.api.Get(ctx, idxPath.immutablePartiallyResolvedPath) + if err == nil { + if getResp.bytes == nil { + webError(w, fmt.Errorf("%q could not be read: %w", idxPath.immutablePartiallyResolvedPath, files.ErrNotReader), http.StatusUnprocessableEntity) + return false + } + idxFile = getResp.bytes } + } - logger.Debugw("serving index.html file", "path", idxPath) + if err == nil { + logger.Debugw("serving index.html file", "path", idxPath.originalRequestedPath) + idxPath.finalResolvedPath = idxPathMd.LastSegment // write to request - success := i.serveFile(ctx, w, r, resolvedPath, idxPath, f, begin) + success := i.serveFile(ctx, w, r, idxPath, idxFile, "text/html", begin) if success { - i.unixfsDirIndexGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsDirIndexGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) } return success - case resolver.ErrNoLink: - logger.Debugw("no index.html; noop", "path", idxPath) - default: + } + + if isErrNotFound(err) { + logger.Debugw("no index.html; noop", "path", idxPath.originalRequestedPath) + } else if err != nil { webError(w, err, http.StatusInternalServerError) return false } @@ -104,8 +131,8 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * // type instead of relying on autodetection (which may fail). w.Header().Set("Content-Type", "text/html") - // Generated dir index requires custom Etag (output may change between go-ipfs versions) - dirEtag := getDirListingEtag(resolvedPath.Cid()) + // Generated dir index requires custom Etag (output may change between go-libipfs versions) + dirEtag := getDirListingEtag(contentPath.finalResolvedPath.Cid()) w.Header().Set("Etag", dirEtag) if r.Method == http.MethodHead { @@ -113,24 +140,22 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * return true } - results, err := i.api.LsUnixFsDir(ctx, resolvedPath) - if err != nil { - webError(w, err, http.StatusInternalServerError) - return false - } - - dirListing := make([]assets.DirectoryItem, 0, len(results)) - for link := range results { - if link.Err != nil { - webError(w, link.Err, http.StatusInternalServerError) + var dirListing []assets.DirectoryItem + for l := range dirResp.directory { + if l.Err != nil { + webError(w, l.Err, http.StatusInternalServerError) return false } - hash := link.Cid.String() + name := l.Link.Name + sz := l.Link.Size + linkCid := l.Link.Cid + + hash := linkCid.String() di := assets.DirectoryItem{ - Size: humanize.Bytes(uint64(link.Size)), - Name: link.Name, - Path: gopath.Join(originalURLPath, link.Name), + Size: humanize.Bytes(sz), + Name: name, + Path: gopath.Join(originalURLPath, name), Hash: hash, ShortHash: assets.ShortHash(hash), } @@ -142,7 +167,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * backLink := originalURLPath // don't go further up than /ipfs/$hash/ - pathSplit := path.SplitList(contentPath.String()) + pathSplit := path.SplitList(contentPath.originalRequestedPath.String()) switch { // skip backlink when listing a content root case len(pathSplit) == 3: // url: /ipfs/$hash @@ -161,13 +186,11 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } } - size := "?" - if s, err := dir.Size(); err == nil { - // Size may not be defined/supported. Continue anyways. - size = humanize.Bytes(uint64(s)) - } + // TODO: Why was size allowed to be undefined/supported with continuing anyways? Is this an oversight? + // Note: If in the future size is not required we can use "?" to denote unknown size + size := humanize.Bytes(dirResp.size) - hash := resolvedPath.Cid().String() + hash := contentPath.finalResolvedPath.Cid().String() // Gateway root URL to be used when linking to other rootIDs. // This will be blank unless subdomain or DNSLink resolution is being used @@ -181,7 +204,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * gwURL = "" } - dnslink := assets.HasDNSLinkOrigin(gwURL, contentPath.String()) + dnslink := assets.HasDNSLinkOrigin(gwURL, contentPath.originalRequestedPath.String()) // See comment above where originalUrlPath is declared. tplData := assets.DirectoryTemplateData{ @@ -189,8 +212,8 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * DNSLink: dnslink, Listing: dirListing, Size: size, - Path: contentPath.String(), - Breadcrumbs: assets.Breadcrumbs(contentPath.String(), dnslink), + Path: contentPath.originalRequestedPath.String(), + Breadcrumbs: assets.Breadcrumbs(contentPath.originalRequestedPath.String(), dnslink), BackLink: backLink, Hash: hash, } @@ -203,7 +226,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } // Update metrics - i.unixfsGenDirListingGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsGenDirListingGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) return true } diff --git a/gateway/handler_unixfs_file.go b/gateway/handler_unixfs_file.go index 55a61ee8c..a1be90d80 100644 --- a/gateway/handler_unixfs_file.go +++ b/gateway/handler_unixfs_file.go @@ -12,22 +12,21 @@ import ( "github.com/gabriel-vasile/mimetype" "github.com/ipfs/go-libipfs/files" - ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) // serveFile returns data behind a file along with HTTP headers based on // the file itself, its CID and the contentPath used for accessing it. -func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, file files.File, begin time.Time) bool { - _, span := spanTrace(ctx, "ServeFile", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) +func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath contentPathRequest, file files.File, fileContentType string, begin time.Time) bool { + _, span := spanTrace(ctx, "ServeFile", trace.WithAttributes(attribute.String("path", contentPath.finalResolvedPath.String()))) defer span.End() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + modtime := addCacheControlHeaders(w, r, contentPath, contentPath.finalResolvedPath.Cid()) // Set Content-Disposition - name := addContentDispositionHeader(w, r, contentPath) + name := addContentDispositionHeader(w, r, contentPath.originalRequestedPath) // Prepare size value for Content-Length HTTP header (set inside of http.ServeContent) size, err := file.Size() @@ -60,6 +59,9 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. ctype = "inode/symlink" } else { ctype = mime.TypeByExtension(gopath.Ext(name)) + if ctype == "" { + ctype = fileContentType + } if ctype == "" { // uses https://github.com/gabriel-vasile/mimetype library to determine the content type. // Fixes https://github.com/ipfs/kubo/issues/7252 @@ -98,7 +100,7 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. // Was response successful? if dataSent { // Update metrics - i.unixfsFileGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsFileGetMetric.WithLabelValues(contentPath.originalRequestedPath.Namespace()).Observe(time.Since(begin).Seconds()) } return dataSent diff --git a/gateway/hostname.go b/gateway/hostname.go index 563804e12..58ac433eb 100644 --- a/gateway/hostname.go +++ b/gateway/hostname.go @@ -57,7 +57,7 @@ type Specification struct { // noDNSLink configures the gateway to _not_ perform DNS TXT record lookups in // response to requests with values in `Host` HTTP header. This flag can be overridden // per FQDN in publicGateways. -func WithHostname(next http.Handler, api API, publicGateways map[string]*Specification, noDNSLink bool) http.HandlerFunc { +func WithHostname(next http.Handler, api IPFSBackend, publicGateways map[string]*Specification, noDNSLink bool) http.HandlerFunc { gateways := prepareHostnameGateways(publicGateways) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -266,7 +266,7 @@ func isDomainNameAndNotPeerID(hostname string) bool { } // hasDNSLinkRecord returns if a DNS TXT record exists for the provided host. -func hasDNSLinkRecord(ctx context.Context, api API, host string) bool { +func hasDNSLinkRecord(ctx context.Context, api IPFSBackend, host string) bool { dnslinkName := stripPort(host) if !isDomainNameAndNotPeerID(dnslinkName) { @@ -353,7 +353,7 @@ func toDNSLinkFQDN(dnsLabel string) (fqdn string) { } // Converts a hostname/path to a subdomain-based URL, if applicable. -func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, api API) (redirURL string, err error) { +func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, api IPFSBackend) (redirURL string, err error) { var scheme, ns, rootID, rest string query := r.URL.RawQuery diff --git a/gateway/path.go b/gateway/path.go new file mode 100644 index 000000000..74c22f65a --- /dev/null +++ b/gateway/path.go @@ -0,0 +1,22 @@ +package gateway + +import ( + ipath "github.com/ipfs/interface-go-ipfs-core/path" +) + +// Path, ResolvedPath, ImmutablePath and other path types → single ContentPath type that has sub-states inside (original, resolved mutable→immutable, and fully resolved to /ipfs/cid + optional remainder) +type contentPathRequest struct { + originalRequestedPath ipath.Path // original path + immutableRequestedPath ImmutablePath // path after resolving mutability + immutablePartiallyResolvedPath ImmutablePath // closest resolution to the final path we have so far + finalResolvedPath ipath.Resolved // the path after it has been resolved down to its final CID + remainder. Note: this may include non-standard IPLD resolution such as following 200s from UnixFS _redirects +} + +func newPath(originalPath ipath.Path) contentPathRequest { + return contentPathRequest{ + originalRequestedPath: originalPath, + immutableRequestedPath: ImmutablePath{}, + immutablePartiallyResolvedPath: ImmutablePath{}, + finalResolvedPath: nil, + } +} diff --git a/go.mod b/go.mod index a48d98dfd..21743f0ab 100644 --- a/go.mod +++ b/go.mod @@ -34,8 +34,8 @@ require ( github.com/ipfs/go-namesys v0.7.0 github.com/ipfs/go-path v0.3.1 github.com/ipfs/go-peertaskqueue v0.8.1 - github.com/ipfs/go-unixfs v0.3.1 - github.com/ipfs/go-unixfsnode v1.5.1 + github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa + github.com/ipfs/go-unixfsnode v1.5.2 github.com/ipfs/interface-go-ipfs-core v0.10.0 github.com/ipld/go-car v0.5.0 github.com/ipld/go-car/v2 v2.5.1 @@ -43,12 +43,15 @@ require ( github.com/ipld/go-ipld-prime v0.19.0 github.com/jbenet/goprocess v0.1.4 github.com/libp2p/go-buffer-pool v0.1.0 + github.com/libp2p/go-doh-resolver v0.4.0 github.com/libp2p/go-libp2p v0.25.1 github.com/libp2p/go-libp2p-record v0.2.0 + github.com/libp2p/go-libp2p-routing-helpers v0.4.0 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-msgio v0.3.0 github.com/miekg/dns v1.1.50 github.com/multiformats/go-multiaddr v0.8.0 + github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multibase v0.1.1 github.com/multiformats/go-multicodec v0.7.0 github.com/multiformats/go-multihash v0.2.1 @@ -79,10 +82,9 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/go-bitfield v1.0.0 // indirect + github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.1.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect - github.com/ipfs/go-ipfs-files v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-ipld-legacy v0.1.1 // indirect @@ -102,7 +104,6 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect diff --git a/go.sum b/go.sum index ede1a802c..a53f62100 100644 --- a/go.sum +++ b/go.sum @@ -342,8 +342,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= -github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= +github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= +github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -388,7 +388,6 @@ github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxp github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= -github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -401,11 +400,7 @@ github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSO github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= -github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= -github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= -github.com/ipfs/go-ipfs-files v0.3.0/go.mod h1:xAUtYMwB+iu/dtf6+muHNSFQCJG2dSiStR2P6sn9tIM= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= -github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= @@ -417,7 +412,6 @@ github.com/ipfs/go-ipfs-routing v0.3.0/go.mod h1:dKqtTFIql7e1zYsEuWLyuOU+E0WJWW8 github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= @@ -425,7 +419,6 @@ github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= @@ -444,7 +437,6 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= github.com/ipfs/go-merkledag v0.9.0 h1:DFC8qZ96Dz1hMT7dtIpcY524eFFDiEWAF8hNJHWW2pk= github.com/ipfs/go-merkledag v0.9.0/go.mod h1:bPHqkHt5OZ0p1n3iqPeDiw2jIBkjAytRjS3WSBwjq90= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -456,10 +448,10 @@ github.com/ipfs/go-path v0.3.1/go.mod h1:eNLsxJEEMxn/CDzUJ6wuNl+6No6tEUhOZcPKsZs github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= -github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8= -github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= -github.com/ipfs/go-unixfsnode v1.5.1 h1:JcR3t5C2nM1V7PMzhJ/Qmo19NkoFIKweDSZyDx+CjkI= -github.com/ipfs/go-unixfsnode v1.5.1/go.mod h1:ed79DaG9IEuZITJVQn4U6MZDftv6I3ygUBLPfhEbHvk= +github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa h1:X8DPpsI3xvdsNxrsHi+ji39rjIvfPna3+XD+iQehbNQ= +github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= +github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew= +github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= @@ -469,7 +461,6 @@ github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= github.com/ipld/go-car/v2 v2.5.1 h1:U2ux9JS23upEgrJScW8VQuxmE94560kYxj9CQUpcfmk= github.com/ipld/go-car/v2 v2.5.1/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= -github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.5.0 h1:RspDRdsJpLfgCI0ONhTAnbHdySGD4t+LHSPK4X1+R0k= github.com/ipld/go-codec-dagpb v1.5.0/go.mod h1:0yRIutEFD8o1DGVqw4RSHh+BUTlJA9XWldxaaWR/o4g= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= @@ -546,6 +537,8 @@ github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0 github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU= github.com/libp2p/go-conn-security-multistream v0.2.1/go.mod h1:cR1d8gA0Hr59Fj6NhaTpFhJZrjSYuNmhpT2r25zYR70= +github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw= +github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVhhBjyhhCJs8= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= @@ -624,6 +617,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= @@ -803,6 +798,7 @@ github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6 github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= +github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= @@ -1062,7 +1058,6 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= -github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -1281,7 +1276,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1518,7 +1512,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=