From dee0ee7e9c7b664a0582c3e5c40e5f03f8b2d2f9 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 24 Mar 2022 08:12:53 -0700 Subject: [PATCH 1/2] Car command supports for `largebytes` nodes and partial unixfs file traversals --- cmd/car/car.go | 2 +- cmd/car/create.go | 2 +- cmd/car/extract.go | 6 +++++- cmd/car/get.go | 13 ++++++++++++- cmd/go.mod | 8 ++++---- cmd/go.sum | 23 +++++++++++++---------- 6 files changed, 36 insertions(+), 18 deletions(-) diff --git a/cmd/car/car.go b/cmd/car/car.go index d34ce416..90cf9425 100644 --- a/cmd/car/car.go +++ b/cmd/car/car.go @@ -130,7 +130,7 @@ func main1() int { }, { Name: "list", - Aliases: []string{"l"}, + Aliases: []string{"l", "ls"}, Usage: "List the CIDs in a car", Action: ListCar, Flags: []cli.Flag{ diff --git a/cmd/car/create.go b/cmd/car/create.go index de9f6abd..3a76c913 100644 --- a/cmd/car/create.go +++ b/cmd/car/create.go @@ -117,7 +117,7 @@ func writeFiles(ctx context.Context, bs *blockstore.ReadWrite, paths ...string) // make a directory for the file(s). - root, err := builder.BuildUnixFSDirectory(topLevel, &ls) + root, _, err := builder.BuildUnixFSDirectory(topLevel, &ls) if err != nil { return cid.Undef, nil } diff --git a/cmd/car/extract.go b/cmd/car/extract.go index 5ebc2b77..ae2da875 100644 --- a/cmd/car/extract.go +++ b/cmd/car/extract.go @@ -239,13 +239,17 @@ func extractFile(c *cli.Context, ls *ipld.LinkSystem, n ipld.Node, outputName st if err != nil { return err } + nlr, err := node.AsLargeBytes() + if err != nil { + return err + } f, err := os.Create(outputName) if err != nil { return err } defer f.Close() - _, err = io.Copy(f, node) + _, err = io.Copy(f, nlr) return err } diff --git a/cmd/car/get.go b/cmd/car/get.go index 090f0fad..4289fc48 100644 --- a/cmd/car/get.go +++ b/cmd/car/get.go @@ -17,6 +17,7 @@ import ( "github.com/ipfs/go-cid" ipfsbs "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-unixfsnode" "github.com/ipld/go-car" "github.com/ipld/go-car/v2/blockstore" "github.com/ipld/go-ipld-prime/datamodel" @@ -128,6 +129,7 @@ func writeCarV2(ctx context.Context, rootCid cid.Cid, output string, bs *blockst } ls := cidlink.DefaultLinkSystem() + ls.KnownReifiers = map[string]linking.NodeReifier{"unixfs": unixfsnode.Reify} ls.TrustedStorage = true ls.StorageReadOpener = func(_ linking.LinkContext, l datamodel.Link) (io.Reader, error) { if cl, ok := l.(cidlink.Link); ok { @@ -176,7 +178,16 @@ func writeCarV2(ctx context.Context, rootCid cid.Cid, output string, bs *blockst return err } - err = traversalProgress.WalkMatching(rootNode, s, func(p traversal.Progress, n datamodel.Node) error { return nil }) + err = traversalProgress.WalkMatching(rootNode, s, func(p traversal.Progress, n datamodel.Node) error { + lb, ok := n.(datamodel.LargeBytesNode) + if ok { + rs, err := lb.AsLargeBytes() + if err == nil { + io.Copy(io.Discard, rs) + } + } + return nil + }) if err != nil { return err } diff --git a/cmd/go.mod b/cmd/go.mod index 74c9c049..0d17ecbb 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -7,14 +7,14 @@ require ( github.com/ipfs/go-block-format v0.0.3 github.com/ipfs/go-cid v0.1.0 github.com/ipfs/go-ipfs-blockstore v1.1.2 - github.com/ipfs/go-unixfsnode v1.2.0 + github.com/ipfs/go-unixfsnode v1.4.0 github.com/ipld/go-car v0.3.3 github.com/ipld/go-car/v2 v2.1.2-0.20220121091436-7e10f104f452 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.4 - github.com/multiformats/go-multicodec v0.4.0 + github.com/ipld/go-ipld-prime v0.16.1-0.20220324131737-6e219a02ca16 + github.com/multiformats/go-multicodec v0.4.1 github.com/multiformats/go-multihash v0.1.0 github.com/multiformats/go-varint v0.0.6 - github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451 + github.com/rogpeppe/go-internal v1.8.1 github.com/urfave/cli/v2 v2.3.0 ) diff --git a/cmd/go.sum b/cmd/go.sum index 79b7eee9..5c20b8f6 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -126,8 +126,9 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -187,8 +188,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -342,8 +344,8 @@ github.com/ipfs/go-peertaskqueue v0.7.0 h1:VyO6G4sbzX80K58N60cCaHsSsypbUNs1GjO5s github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8= github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= -github.com/ipfs/go-unixfsnode v1.2.0 h1:tHHBJftsJyHGa8bS62PpkYNqHy/Sug3c/vxxC8NaGQY= -github.com/ipfs/go-unixfsnode v1.2.0/go.mod h1:mQEgLjxkV/1mohkC4p7taRRBYPBeXu97SA3YaerT2q0= +github.com/ipfs/go-unixfsnode v1.4.0 h1:9BUxHBXrbNi8mWHc6j+5C580WJqtVw9uoeEKn4tMhwA= +github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipld/go-car v0.3.3 h1:D6y+jvg9h2ZSv7GLUMWUwg5VTLy1E7Ak+uQw5orOg3I= @@ -357,8 +359,9 @@ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvB github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.14.3-0.20211207234443-319145880958/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= -github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc= -github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= +github.com/ipld/go-ipld-prime v0.16.1-0.20220324131737-6e219a02ca16 h1:KCFEByXqCF5kymLHX3L/6AG3DNgKHQrUklE2nRFHcC4= +github.com/ipld/go-ipld-prime v0.16.1-0.20220324131737-6e219a02ca16/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -683,8 +686,8 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.4.0 h1:fbqb6ky7erjdD+/zaEBJgZWu1i8D6i/wmPywGK7sdow= -github.com/multiformats/go-multicodec v0.4.0/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.4.1 h1:BSJbf+zpghcZMZrwTYBGwy0CPcVZGWiC72Cp8bBd4R4= +github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -796,8 +799,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451 h1:d1PiN4RxzIFXCJTvRkvSkKqwtRAl5ZV4lATKtQI0B7I= -github.com/rogpeppe/go-internal v1.8.1-0.20210923151022-86f73c517451/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= From b27f5e059ba6492c17368e39de0b75de341787ba Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 24 Mar 2022 08:20:35 -0700 Subject: [PATCH 2/2] check copy error --- cmd/car/get.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/car/get.go b/cmd/car/get.go index 4289fc48..84531b7a 100644 --- a/cmd/car/get.go +++ b/cmd/car/get.go @@ -183,7 +183,10 @@ func writeCarV2(ctx context.Context, rootCid cid.Cid, output string, bs *blockst if ok { rs, err := lb.AsLargeBytes() if err == nil { - io.Copy(io.Discard, rs) + _, err := io.Copy(io.Discard, rs) + if err != nil { + return err + } } } return nil