filestore util: Add 'filestore rm' command.

License: MIT
Signed-off-by: Kevin Atkinson <[email protected]>

core/commands/filestore.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io"
 
+	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
+	butil "github.com/ipfs/go-ipfs/blocks/blockstore/util"
 	cmds "github.com/ipfs/go-ipfs/commands"
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/filestore"
@@ -20,6 +22,7 @@ var FileStoreCmd = &cmds.Command{
 		"ls":     lsFileStore,
 		"verify": verifyFileStore,
 		"dups":   dupsFileStore,
+		"rm":     rmFileStore,
 	},
 }
 
@@ -201,6 +204,60 @@ var dupsFileStore = &cmds.Command{
 	Type:       RefWrapper{},
 }
 
+var rmFileStore = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Remove IPFS block(s) from just the filestore or blockstore.",
+		ShortDescription: `
+Remove blocks from either the filestore or the main blockstore.
+`,
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("hash", true, true, "CID's of block(s) to remove."),
+	},
+	Options: []cmds.Option{
+		cmds.BoolOption("force", "f", "Ignore nonexistent blocks."),
+		cmds.BoolOption("quiet", "q", "Write minimal output."),
+		cmds.BoolOption("non-filestore", "Remove non-filestore blocks"),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		n, fs, err := getFilestore(req)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		hashes := req.Arguments()
+		force, _, _ := req.Option("force").Bool()
+		quiet, _, _ := req.Option("quiet").Bool()
+		nonFilestore, _, _ := req.Option("non-filestore").Bool()
+		prefix := filestore.FilestorePrefix.String()
+		if nonFilestore {
+			prefix = bs.BlockPrefix.String()
+		}
+		cids := make([]*cid.Cid, 0, len(hashes))
+		for _, hash := range hashes {
+			c, err := cid.Decode(hash)
+			if err != nil {
+				res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmds.ErrNormal)
+				return
+			}
+
+			cids = append(cids, c)
+		}
+		ch, err := filestore.RmBlocks(fs, n.Blockstore, n.Pinning, cids, butil.RmBlocksOpts{
+			Prefix: prefix,
+			Quiet:  quiet,
+			Force:  force,
+		})
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		res.SetOutput(ch)
+	},
+	PostRun: blockRmCmd.PostRun,
+	Type:    butil.RemovedBlock{},
+}
+
 func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error) {
 	n, err := req.InvocContext().GetNode()
 	if err != nil {

filestore/remove.go

@@ -0,0 +1,91 @@
+package filestore
+
+import (
+	"fmt"
+
+	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
+	u "github.com/ipfs/go-ipfs/blocks/blockstore/util"
+	"github.com/ipfs/go-ipfs/pin"
+
+	cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
+	ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
+)
+
+// RmBlocks removes blocks from either the filestore or the
+// blockstore.  It is similar to blockstore_util.RmBlocks but allows
+// the removal of pinned block from one store if it is also in the
+// other.
+func RmBlocks(fs *Filestore, lock bs.GCLocker, pins pin.Pinner, cids []*cid.Cid, opts u.RmBlocksOpts) (<-chan interface{}, error) {
+	// make the channel large enough to hold any result to avoid
+	// blocking while holding the GCLock
+	out := make(chan interface{}, len(cids))
+
+	var blocks deleter
+	switch opts.Prefix {
+	case FilestorePrefix.String():
+		blocks = fs.fm
+	case bs.BlockPrefix.String():
+		blocks = fs.bs
+	default:
+		return nil, fmt.Errorf("unknown prefix: %s", opts.Prefix)
+	}
+
+	go func() {
+		defer close(out)
+
+		unlocker := lock.GCLock()
+		defer unlocker.Unlock()
+
+		stillOkay := filterPinned(fs, pins, out, cids, blocks)
+
+		for _, c := range stillOkay {
+			err := blocks.DeleteBlock(c)
+			if err != nil && opts.Force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
+				// ignore non-existent blocks
+			} else if err != nil {
+				out <- &u.RemovedBlock{Hash: c.String(), Error: err.Error()}
+			} else if !opts.Quiet {
+				out <- &u.RemovedBlock{Hash: c.String()}
+			}
+		}
+	}()
+	return out, nil
+}
+
+type deleter interface {
+	DeleteBlock(c *cid.Cid) error
+}
+
+func filterPinned(fs *Filestore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, foundIn deleter) []*cid.Cid {
+	stillOkay := make([]*cid.Cid, 0, len(cids))
+	res, err := pins.CheckIfPinned(cids...)
+	if err != nil {
+		out <- &u.RemovedBlock{Error: fmt.Sprintf("pin check failed: %s", err)}
+		return nil
+	}
+	for _, r := range res {
+		if !r.Pinned() || availableElsewhere(fs, foundIn, r.Key) {
+			stillOkay = append(stillOkay, r.Key)
+		} else {
+			out <- &u.RemovedBlock{
+				Hash:  r.Key.String(),
+				Error: r.String(),
+			}
+		}
+	}
+	return stillOkay
+}
+
+func availableElsewhere(fs *Filestore, foundIn deleter, c *cid.Cid) bool {
+	switch {
+	case fs.fm == foundIn:
+		have, _ := fs.bs.Has(c)
+		return have
+	case fs.bs == foundIn:
+		have, _ := fs.fm.Has(c)
+		return have
+	default:
+		// programmer error
+		panic("invalid pointer for foundIn")
+	}
+}

test/sharness/t0271-filestore-utils.sh

@@ -153,7 +153,7 @@ test_filestore_verify() {
   test_init_dataset
 }
 
-test_filestore_dups() {
+test_filestore_dups_and_rm() {
   # make sure the filestore is in a clean state
   test_filestore_state
 
@@ -163,6 +163,28 @@ test_filestore_dups() {
     echo "$FILE1_HASH" > dups_expect
     test_cmp dups_expect dups_actual
   '
+
+  test_expect_success "remove non-filestore block of dup ok" '
+    ipfs filestore rm --non-filestore $FILE1_HASH &&
+    ipfs filestore dups > dups_actual &&
+    test_cmp /dev/null dups_actual
+  '
+
+  test_expect_success "block still in filestore" '
+    ipfs filestore ls $FILE1_HASH | grep -q file1
+  '
+
+  test_expect_success "remove non-duplicate pinned block not ok" '
+    test_must_fail ipfs filestore rm $FILE1_HASH 2>&1 | tee rm_err &&
+    grep -q pinned rm_err
+  '
+
+  test_expect_success "remove filestore block of dup ok" '
+    ipfs add --raw-leaves somedir/file1 &&
+    ipfs filestore rm $FILE1_HASH &&
+    ipfs filestore dups > dups_actual &&
+    test_cmp /dev/null dups_actual
+  '
 }
 
 #
@@ -175,7 +197,7 @@ test_filestore_adds
 
 test_filestore_verify
 
-test_filestore_dups
+test_filestore_dups_and_rm
 
 #
 # With daemon
@@ -191,7 +213,7 @@ test_filestore_adds
 
 test_filestore_verify
 
-test_filestore_dups
+test_filestore_dups_and_rm
 
 test_kill_ipfs_daemon