diff --git a/core/commands/filestore.go b/core/commands/filestore.go index b30d3ba8036..09eea770d4a 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -6,6 +6,8 @@ import ( "io" "os" + bs "github.com/ipfs/go-ipfs/blocks/blockstore" + butil "github.com/ipfs/go-ipfs/blocks/blockstore/util" oldCmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" @@ -22,6 +24,7 @@ var FileStoreCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ "ls": lsFileStore, + "rm": rmFileStore, }, OldSubcommands: map[string]*oldCmds.Command{ "verify": verifyFileStore, @@ -251,6 +254,63 @@ var dupsFileStore = &oldCmds.Command{ Type: RefWrapper{}, } +var rmFileStore = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Remove IPFS block(s) from just the filestore or blockstore.", + ShortDescription: ` +Remove blocks from either the filestore or the main blockstore. +`, + }, + Arguments: []cmdkit.Argument{ + cmdkit.StringArg("hash", true, true, "CID's of block(s) to remove."), + }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("force", "f", "Ignore nonexistent blocks."), + cmdkit.BoolOption("quiet", "q", "Write minimal output."), + cmdkit.BoolOption("non-filestore", "Remove non-filestore blocks"), + }, + Run: func(req cmds.Request, res cmds.ResponseEmitter) { + n, fs, err := getFilestore(req.InvocContext()) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + hashes := req.Arguments() + force, _, _ := req.Option("force").Bool() + quiet, _, _ := req.Option("quiet").Bool() + nonFilestore, _, _ := req.Option("non-filestore").Bool() + prefix := filestore.FilestorePrefix.String() + if nonFilestore { + prefix = bs.BlockPrefix.String() + } + cids := make([]*cid.Cid, 0, len(hashes)) + for _, hash := range hashes { + c, err := cid.Decode(hash) + if err != nil { + res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmdkit.ErrNormal) + return + } + + cids = append(cids, c) + } + ch, err := filestore.RmBlocks(fs, n.Blockstore, n.Pinning, cids, butil.RmBlocksOpts{ + Prefix: prefix, + Quiet: quiet, + Force: force, + }) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + err = res.Emit(ch) + if err != nil { + log.Error(err) + } + }, + PostRun: blockRmCmd.PostRun, + Type: butil.RemovedBlock{}, +} + type getNoder interface { GetNode() (*core.IpfsNode, error) } diff --git a/filestore/remove.go b/filestore/remove.go new file mode 100644 index 00000000000..594bc97a4ea --- /dev/null +++ b/filestore/remove.go @@ -0,0 +1,91 @@ +package filestore + +import ( + "fmt" + + bs "github.com/ipfs/go-ipfs/blocks/blockstore" + u "github.com/ipfs/go-ipfs/blocks/blockstore/util" + "github.com/ipfs/go-ipfs/pin" + + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" +) + +// RmBlocks removes blocks from either the filestore or the +// blockstore. It is similar to blockstore_util.RmBlocks but allows +// the removal of pinned block from one store if it is also in the +// other. +func RmBlocks(fs *Filestore, lock bs.GCLocker, pins pin.Pinner, cids []*cid.Cid, opts u.RmBlocksOpts) (<-chan interface{}, error) { + // make the channel large enough to hold any result to avoid + // blocking while holding the GCLock + out := make(chan interface{}, len(cids)) + + var blocks deleter + switch opts.Prefix { + case FilestorePrefix.String(): + blocks = fs.fm + case bs.BlockPrefix.String(): + blocks = fs.bs + default: + return nil, fmt.Errorf("unknown prefix: %s", opts.Prefix) + } + + go func() { + defer close(out) + + unlocker := lock.GCLock() + defer unlocker.Unlock() + + stillOkay := filterPinned(fs, pins, out, cids, blocks) + + for _, c := range stillOkay { + err := blocks.DeleteBlock(c) + if err != nil && opts.Force && (err == bs.ErrNotFound || err == ds.ErrNotFound) { + // ignore non-existent blocks + } else if err != nil { + out <- &u.RemovedBlock{Hash: c.String(), Error: err.Error()} + } else if !opts.Quiet { + out <- &u.RemovedBlock{Hash: c.String()} + } + } + }() + return out, nil +} + +type deleter interface { + DeleteBlock(c *cid.Cid) error +} + +func filterPinned(fs *Filestore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, foundIn deleter) []*cid.Cid { + stillOkay := make([]*cid.Cid, 0, len(cids)) + res, err := pins.CheckIfPinned(cids...) + if err != nil { + out <- &u.RemovedBlock{Error: fmt.Sprintf("pin check failed: %s", err)} + return nil + } + for _, r := range res { + if !r.Pinned() || availableElsewhere(fs, foundIn, r.Key) { + stillOkay = append(stillOkay, r.Key) + } else { + out <- &u.RemovedBlock{ + Hash: r.Key.String(), + Error: r.String(), + } + } + } + return stillOkay +} + +func availableElsewhere(fs *Filestore, foundIn deleter, c *cid.Cid) bool { + switch { + case fs.fm == foundIn: + have, _ := fs.bs.Has(c) + return have + case fs.bs == foundIn: + have, _ := fs.fm.Has(c) + return have + default: + // programmer error + panic("invalid pointer for foundIn") + } +} diff --git a/test/sharness/t0271-filestore-utils.sh b/test/sharness/t0271-filestore-utils.sh index 9105c2f8341..ade90402d32 100755 --- a/test/sharness/t0271-filestore-utils.sh +++ b/test/sharness/t0271-filestore-utils.sh @@ -153,7 +153,7 @@ test_filestore_verify() { test_init_dataset } -test_filestore_dups() { +test_filestore_dups_and_rm() { # make sure the filestore is in a clean state test_filestore_state @@ -163,6 +163,28 @@ test_filestore_dups() { echo "$FILE1_HASH" > dups_expect test_cmp dups_expect dups_actual ' + + test_expect_success "remove non-filestore block of dup ok" ' + ipfs filestore rm --non-filestore $FILE1_HASH && + ipfs filestore dups > dups_actual && + test_cmp /dev/null dups_actual + ' + + test_expect_success "block still in filestore" ' + ipfs filestore ls $FILE1_HASH | grep -q file1 + ' + + test_expect_success "remove non-duplicate pinned block not ok" ' + test_must_fail ipfs filestore rm $FILE1_HASH 2>&1 | tee rm_err && + grep -q pinned rm_err + ' + + test_expect_success "remove filestore block of dup ok" ' + ipfs add --raw-leaves somedir/file1 && + ipfs filestore rm $FILE1_HASH && + ipfs filestore dups > dups_actual && + test_cmp /dev/null dups_actual + ' } # @@ -175,7 +197,7 @@ test_filestore_adds test_filestore_verify -test_filestore_dups +test_filestore_dups_and_rm # # With daemon @@ -191,7 +213,7 @@ test_filestore_adds test_filestore_verify -test_filestore_dups +test_filestore_dups_and_rm test_kill_ipfs_daemon