diff --git a/assets/assets.go b/assets/assets.go index a47b7c82a068..754160f21661 100644 --- a/assets/assets.go +++ b/assets/assets.go @@ -80,13 +80,9 @@ func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) { return nil, err } - if err := nd.Pinning.Pin(nd.Context(), dir, true); err != nil { + if err := nd.Pinning.Pin(nd.Context(), "assets", dir, true); err != nil { return nil, fmt.Errorf("assets: Pinning on init-docu failed: %s", err) } - if err := nd.Pinning.Flush(); err != nil { - return nil, fmt.Errorf("assets: Pinning flush failed: %s", err) - } - return dir.Cid(), nil } diff --git a/core/builder.go b/core/builder.go index fb28449e4969..fd1a32d018e1 100644 --- a/core/builder.go +++ b/core/builder.go @@ -258,15 +258,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { n.Blocks = bserv.New(n.Blockstore, n.Exchange) n.DAG = dag.NewDAGService(n.Blocks) - internalDag := dag.NewDAGService(bserv.New(n.Blockstore, offline.Exchange(n.Blockstore))) - n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.DAG, internalDag) - if err != nil { - // TODO: we should move towards only running 'NewPinner' explicitly on - // node init instead of implicitly here as a result of the pinner keys - // not being found in the datastore. - // this is kinda sketchy and could cause data loss - n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.DAG, internalDag) - } + n.Pinning = pin.NewPinner(n.DAG, n.Repo.Datastore()) n.Resolver = resolver.NewBasicResolver(n.DAG) if cfg.Online { diff --git a/core/commands/add.go b/core/commands/add.go index 8695695f4c76..d31841670ff6 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -41,6 +41,7 @@ const ( onlyHashOptionName = "only-hash" chunkerOptionName = "chunker" pinOptionName = "pin" + pinPathOptionName = "pinpath" rawLeavesOptionName = "raw-leaves" noCopyOptionName = "nocopy" fstoreCacheOptionName = "fscache" @@ -119,6 +120,7 @@ You can now check what blocks have been created by: cmdkit.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."), cmdkit.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes] or rabin-[min]-[avg]-[max]").WithDefault("size-262144"), cmdkit.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true), + cmdkit.StringOption(pinPathOptionName, "P", "Pin object under this path.").WithDefault("added/"), cmdkit.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"), cmdkit.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"), cmdkit.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), @@ -174,6 +176,7 @@ You can now check what blocks have been created by: silent, _ := req.Options[silentOptionName].(bool) chunker, _ := req.Options[chunkerOptionName].(string) dopin, _ := req.Options[pinOptionName].(bool) + pinPath, _ := req.Options[pinPathOptionName].(string) rawblks, rbset := req.Options[rawLeavesOptionName].(bool) nocopy, _ := req.Options[noCopyOptionName].(bool) fscache, _ := req.Options[fstoreCacheOptionName].(bool) @@ -278,6 +281,7 @@ You can now check what blocks have been created by: } fileAdder.Out = outChan + fileAdder.PinPath = pinPath fileAdder.Chunker = chunker fileAdder.Progress = progress fileAdder.Hidden = hidden diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 1de0effb2435..51f09141faa2 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -10,7 +10,6 @@ import ( cmds "github.com/ipfs/go-ipfs/commands" e "github.com/ipfs/go-ipfs/core/commands/e" coredag "github.com/ipfs/go-ipfs/core/coredag" - pin "github.com/ipfs/go-ipfs/pin" path "gx/ipfs/QmTKaiDxQqVxmA1bRipSuP7hnTSgnMSmEa98NYeS6fcoiv/go-path" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" @@ -62,7 +61,7 @@ into an object of the specified format. Options: []cmdkit.Option{ cmdkit.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"), cmdkit.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"), - cmdkit.BoolOption("pin", "Pin this object when adding."), + cmdkit.StringOption("pin", "Pin this object when adding.").WithDefault(""), cmdkit.StringOption("hash", "Hash function to use").WithDefault(""), }, Run: func(req cmds.Request, res cmds.Response) { @@ -75,7 +74,7 @@ into an object of the specified format. ienc, _, _ := req.Option("input-enc").String() format, _, _ := req.Option("format").String() hash, _, err := req.Option("hash").String() - dopin, _, err := req.Option("pin").Bool() + pinpath, _, err := req.Option("pin").String() if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -140,15 +139,12 @@ into an object of the specified format. return err } - if dopin { - defer n.Blockstore.PinLock().Unlock() + if pinpath != "" { - cids.ForEach(func(c *cid.Cid) error { - n.Pinning.PinWithMode(c, pin.Recursive) - return nil + err := cids.ForEach(func(c *cid.Cid) error { + return n.Pinning.AddPin(pinpath, c, true) }) - err := n.Pinning.Flush() if err != nil { return err } diff --git a/core/commands/object/object.go b/core/commands/object/object.go index 3edfd9044ed9..1016a5555c80 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -161,7 +161,7 @@ multihash. } } - out := Object{ + out := &Object{ Hash: rp.Cid().String(), Links: outLinks, } @@ -437,6 +437,7 @@ And then run: cmdkit.StringOption("inputenc", "Encoding type of input data. One of: {\"protobuf\", \"json\"}.").WithDefault("json"), cmdkit.StringOption("datafieldenc", "Encoding type of the data field, either \"text\" or \"base64\".").WithDefault("text"), cmdkit.BoolOption("pin", "Pin this object when adding."), + cmdkit.StringOption("pinpath", "Pin under this path").WithDefault("added/"), cmdkit.BoolOption("quiet", "q", "Write minimal output."), }, Run: func(req oldcmds.Request, res oldcmds.Response) { @@ -464,7 +465,13 @@ And then run: return } - dopin, _, err := req.Option("pin").Bool() + pinpath, _, err := req.Option("pinpath").String() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + pin, _, err := req.Option("pin").Bool() if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -473,7 +480,8 @@ And then run: p, err := api.Object().Put(req.Context(), input, options.Object.DataType(datafieldenc), options.Object.InputEnc(inputenc), - options.Object.Pin(dopin)) + options.Object.Pin(pin), + options.Object.PinPath(pinpath)) if err != nil { res.SetError(err, cmdkit.ErrNormal) return diff --git a/core/commands/pin.go b/core/commands/pin.go index 68137c650151..9039d2d0cccc 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -54,11 +54,12 @@ var addPinCmd = &cmds.Command{ }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("ipfs-path", true, true, "Path to object(s) to be pinned.").EnableStdin(), + cmdkit.StringArg("ipfs-path", true, true, "Path(es) to object(s) to be pinned.").EnableStdin(), }, Options: []cmdkit.Option{ - cmdkit.BoolOption("recursive", "r", "Recursively pin the object linked to by the specified object(s).").WithDefault(true), + cmdkit.BoolOption("direct", "d", "Pin the object directly"), cmdkit.BoolOption("progress", "Show progress"), + cmdkit.StringOption("pinpath", "P", "Pin path.").WithDefault("default/"), }, Type: AddPinOutput{}, Run: func(req cmds.Request, res cmds.Response) { @@ -70,16 +71,26 @@ var addPinCmd = &cmds.Command{ defer n.Blockstore.PinLock().Unlock() - // set recursive flag - recursive, _, err := req.Option("recursive").Bool() + // set direct flag + direct, _, err := req.Option("direct").Bool() if err != nil { res.SetError(err, cmdkit.ErrNormal) return } showProgress, _, _ := req.Option("progress").Bool() + args := req.Arguments() + + pinPath, _, err := req.Option("pinpath").String() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + toPin := args + if !showProgress { - added, err := corerepo.Pin(n, req.Context(), req.Arguments(), recursive) + added, err := corerepo.Pin(n, req.Context(), pinPath, toPin, !direct) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -99,7 +110,7 @@ var addPinCmd = &cmds.Command{ } ch := make(chan pinResult, 1) go func() { - added, err := corerepo.Pin(n, ctx, req.Arguments(), recursive) + added, err := corerepo.Pin(n, ctx, pinPath, toPin, !direct) ch <- pinResult{pins: added, err: err} }() @@ -154,8 +165,8 @@ var addPinCmd = &cmds.Command{ } var pintype string - rec, found, _ := res.Request().Option("recursive").Bool() - if rec || !found { + direct, found, _ := res.Request().Option("direct").Bool() + if !direct || !found { pintype = "recursively" } else { pintype = "directly" @@ -175,16 +186,15 @@ var rmPinCmd = &cmds.Command{ Tagline: "Remove pinned objects from local storage.", ShortDescription: ` Removes the pin from the given object allowing it to be garbage -collected if needed. (By default, recursively. Use -r=false for direct pins.) +collected if needed. By default, removes recursive pins. `, }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("ipfs-path", true, true, "Path to object(s) to be unpinned.").EnableStdin(), + cmdkit.StringArg("pin-path", true, true, "Pin paths").EnableStdin(), }, Options: []cmdkit.Option{ - cmdkit.BoolOption("recursive", "r", "Recursively unpin the object linked to by the specified object(s).").WithDefault(true), - cmdkit.BoolOption("explain", "e", "Check for other pinned objects which could cause specified object(s) to be indirectly pinned").WithDefault(false), + cmdkit.BoolOption("direct", "d", "Unpins a direct pin").WithDefault(false), }, Type: PinOutput{}, Run: func(req cmds.Request, res cmds.Response) { @@ -195,25 +205,20 @@ collected if needed. (By default, recursively. Use -r=false for direct pins.) } // set recursive flag - recursive, _, err := req.Option("recursive").Bool() + direct, _, err := req.Option("direct").Bool() + recursive := !direct if err != nil { res.SetError(err, cmdkit.ErrNormal) return } - explain, _, err := req.Option("explain").Bool() + removed, err := corerepo.Unpin(n, req.Context(), req.Arguments(), recursive) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } - removed, err := corerepo.Unpin(n, req.Context(), req.Arguments(), recursive, explain) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - res.SetOutput(&PinOutput{cidsToStrings(removed)}) + res.SetOutput(&PinOutput{removed}) }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { @@ -281,10 +286,12 @@ Example: }, Arguments: []cmdkit.Argument{ + // TODO: implement path listing cmdkit.StringArg("ipfs-path", false, true, "Path to object(s) to be listed."), }, Options: []cmdkit.Option{ cmdkit.StringOption("type", "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\".").WithDefault("all"), + cmdkit.BoolOption("recursive", "r", "List all pins under given prefix"), cmdkit.BoolOption("quiet", "q", "Write just hashes of objects."), }, Run: func(req cmds.Request, res cmds.Response) { @@ -300,6 +307,12 @@ Example: return } + recursive, _, err := req.Option("recursive").Bool() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + switch typeStr { case "all", "direct", "indirect", "recursive": default: @@ -308,18 +321,18 @@ Example: return } - var keys map[string]RefKeyObject - - if len(req.Arguments()) > 0 { - keys, err = pinLsKeys(req.Context(), req.Arguments(), typeStr, n) - } else { - keys, err = pinLsAll(req.Context(), typeStr, n) + args := req.Arguments() + prefix := "" + if len(args) == 1 { + prefix = args[0] } + keys, err := pinLsAll(req.Context(), typeStr, prefix, recursive, n) + if err != nil { res.SetError(err, cmdkit.ErrNormal) } else { - res.SetOutput(&RefKeyList{Keys: keys}) + res.SetOutput(keys) } }, Type: RefKeyList{}, @@ -340,11 +353,11 @@ Example: return nil, e.TypeErr(keys, v) } out := new(bytes.Buffer) - for k, v := range keys.Keys { + for _, v := range keys.Keys { if quiet { - fmt.Fprintf(out, "%s\n", k) + fmt.Fprintf(out, "%s\n", v.Object.String()) } else { - fmt.Fprintf(out, "%s %s\n", k, v.Type) + fmt.Fprintf(out, "%s %s %s\n", v.Object.String(), v.PinType, v.PinPath) } } return out, nil @@ -363,13 +376,11 @@ new pin and removing the old one. }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("from-path", true, false, "Path to old object."), + cmdkit.StringArg("pinpath", true, false, "Pin path of the old object."), cmdkit.StringArg("to-path", true, false, "Path to new object to be pinned."), }, - Options: []cmdkit.Option{ - cmdkit.BoolOption("unpin", "Remove the old pin.").WithDefault(true), - }, - Type: PinOutput{}, + Options: []cmdkit.Option{}, + Type: PinOutput{}, Run: func(req cmds.Request, res cmds.Response) { n, err := req.InvocContext().GetNode() if err != nil { @@ -377,17 +388,7 @@ new pin and removing the old one. return } - unpin, _, err := req.Option("unpin").Bool() - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - from, err := path.ParsePath(req.Arguments()[0]) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } + from := req.Arguments()[0] to, err := path.ParsePath(req.Arguments()[1]) if err != nil { @@ -400,31 +401,24 @@ new pin and removing the old one. ResolveOnce: uio.ResolveUnixfsOnce, } - fromc, err := core.ResolveToCid(req.Context(), n.Namesys, r, from) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - toc, err := core.ResolveToCid(req.Context(), n.Namesys, r, to) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } - err = n.Pinning.Update(req.Context(), fromc, toc, unpin) + err = n.Pinning.Update(req.Context(), from, toc) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } - err = n.Pinning.Flush() + res.SetOutput(&PinOutput{Pins: []string{from, to.String()}}) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } - res.SetOutput(&PinOutput{Pins: []string{from.String(), to.String()}}) }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { @@ -501,19 +495,16 @@ var verifyPinCmd = &cmds.Command{ } type RefKeyObject struct { - Type string + PinType string + PinPath string + Object *cid.Cid } type RefKeyList struct { - Keys map[string]RefKeyObject + Keys []RefKeyObject } -func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsNode) (map[string]RefKeyObject, error) { - - mode, ok := pin.StringToMode(typeStr) - if !ok { - return nil, fmt.Errorf("invalid pin mode '%s'", typeStr) - } +func pinFindCids(args []string, ctx context.Context, n *core.IpfsNode) (map[string]RefKeyObject, error) { keys := make(map[string]RefKeyObject) @@ -522,6 +513,7 @@ func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsN ResolveOnce: uio.ResolveUnixfsOnce, } + cids := make([]*cid.Cid, 0, len(args)) for _, p := range args { pth, err := path.ParsePath(p) if err != nil { @@ -533,58 +525,100 @@ func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsN return nil, err } - pinType, pinned, err := n.Pinning.IsPinnedWithType(c, mode) - if err != nil { - return nil, err - } + cids = append(cids, c) + } - if !pinned { + pinneds, err := n.Pinning.CheckIfPinned(cids...) + if err != nil { + return nil, err + } + + for _, p := range pinneds { + + if !p.Pinned() { return nil, fmt.Errorf("path '%s' is not pinned", p) } - switch pinType { - case "direct", "indirect", "recursive", "internal": - default: - pinType = "indirect through " + pinType - } - keys[c.String()] = RefKeyObject{ - Type: pinType, + keys[p.Key.String()] = RefKeyObject{ + PinType: p.String(), } } return keys, nil } -func pinLsAll(ctx context.Context, typeStr string, n *core.IpfsNode) (map[string]RefKeyObject, error) { +func pinLsAll(ctx context.Context, typeStr string, prefix string, recursive bool, n *core.IpfsNode) (*RefKeyList, error) { + // TODO: replace string with type? + keys := make([]RefKeyObject, 0) - keys := make(map[string]RefKeyObject) + var recursiveMap map[string]*cid.Cid + var directMap map[string]*cid.Cid - AddToResultKeys := func(keyList []*cid.Cid, typeStr string) { - for _, c := range keyList { - keys[c.String()] = RefKeyObject{ - Type: typeStr, - } + if recursive { + var err error + recursiveMap, err = n.Pinning.PrefixedPins(prefix, true) + if err != nil { + return nil, err + } + directMap, err = n.Pinning.PrefixedPins(prefix, false) + if err != nil { + return nil, err + } + } else { + recursiveMap = make(map[string]*cid.Cid) + c, err := n.Pinning.GetPin(prefix, true) + if err != pin.ErrNotPinned && err != nil { + return nil, err + } else if err != pin.ErrNotPinned { + recursiveMap[prefix] = c + } + directMap = make(map[string]*cid.Cid) + c, err = n.Pinning.GetPin(prefix, false) + if err != pin.ErrNotPinned && err != nil { + return nil, err + } else if err != pin.ErrNotPinned { + directMap[prefix] = c } } + if typeStr == "recursive" || typeStr == "all" { + for path, c := range recursiveMap { + // TODO: review whole RefKeyObject thing + keys = append(keys, RefKeyObject{ + PinType: "recursive", + Object: c, + PinPath: path, + }) + } + } if typeStr == "direct" || typeStr == "all" { - AddToResultKeys(n.Pinning.DirectKeys(), "direct") + for path, c := range directMap { + // TODO: review whole RefKeyObject thing + keys = append(keys, RefKeyObject{ + PinType: "direct", + Object: c, + PinPath: path, + }) + } } if typeStr == "indirect" || typeStr == "all" { - set := cid.NewSet() - for _, k := range n.Pinning.RecursiveKeys() { - err := dag.EnumerateChildren(ctx, dag.GetLinksWithDAG(n.DAG), k, set.Visit) + for p, c := range recursiveMap { + set := cid.NewSet() + err := dag.EnumerateChildren(ctx, dag.GetLinksWithDAG(n.DAG), c, set.Visit) if err != nil { return nil, err } + for _, k := range set.Keys() { + keys = append(keys, RefKeyObject{ + PinType: "indirect", + Object: k, + PinPath: p, + }) + } } - AddToResultKeys(set.Keys(), "indirect") - } - if typeStr == "recursive" || typeStr == "all" { - AddToResultKeys(n.Pinning.RecursiveKeys(), "recursive") } - return keys, nil + return &RefKeyList{Keys: keys}, nil } // PinVerifyRes is the result returned for each pin checked in "pin verify" @@ -612,11 +646,19 @@ type pinVerifyOpts struct { func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts) <-chan interface{} { visited := make(map[string]PinStatus) - bs := n.Blocks.Blockstore() DAG := dag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) getLinks := dag.GetLinksWithDAG(DAG) - recPins := n.Pinning.RecursiveKeys() + + recPins, err := n.Pinning.PinnedCids(true) + if err != nil { + status := PinStatus{Ok: false} + // TODO: do something about this error reporting + status.BadNodes = []BadNode{BadNode{Err: err.Error()}} + out := make(chan interface{}) + out <- &PinVerifyRes{"", status} + return out + } var checkPin func(root *cid.Cid) PinStatus checkPin = func(root *cid.Cid) PinStatus { diff --git a/core/coreapi/interface/options/object.go b/core/coreapi/interface/options/object.go index e484a9f36326..3847e3a64f75 100644 --- a/core/coreapi/interface/options/object.go +++ b/core/coreapi/interface/options/object.go @@ -8,6 +8,7 @@ type ObjectPutSettings struct { InputEnc string DataType string Pin bool + PinPath string } type ObjectAddLinkSettings struct { @@ -37,6 +38,7 @@ func ObjectPutOptions(opts ...ObjectPutOption) (*ObjectPutSettings, error) { InputEnc: "json", DataType: "text", Pin: false, + PinPath: "added/", } for _, opt := range opts { @@ -114,6 +116,14 @@ func (objectOpts) Pin(pin bool) ObjectPutOption { } } +// PinPath is an option for Object.Put which specifies under which path to pin the object, default is "added/" +func (objectOpts) PinPath(pinPath string) ObjectPutOption { + return func(settings *ObjectPutSettings) error { + settings.PinPath = pinPath + return nil + } +} + // Create is an option for Object.AddLink which specifies whether create required // directories for the child func (objectOpts) Create(create bool) ObjectAddLinkOption { diff --git a/core/coreapi/interface/options/pin.go b/core/coreapi/interface/options/pin.go index 9d1107f927db..e5ddb7ce5bad 100644 --- a/core/coreapi/interface/options/pin.go +++ b/core/coreapi/interface/options/pin.go @@ -2,14 +2,15 @@ package options type PinAddSettings struct { Recursive bool + PinPath string } type PinLsSettings struct { - Type string + Type string + Recursive bool } type PinUpdateSettings struct { - Unpin bool } type PinAddOption func(*PinAddSettings) error @@ -19,6 +20,7 @@ type PinUpdateOption func(*PinUpdateSettings) error func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { options := &PinAddSettings{ Recursive: true, + PinPath: "default/", } for _, opt := range opts { @@ -33,7 +35,8 @@ func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { options := &PinLsSettings{ - Type: "all", + Type: "all", + Recursive: false, } for _, opt := range opts { @@ -47,9 +50,7 @@ func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { } func PinUpdateOptions(opts ...PinUpdateOption) (*PinUpdateSettings, error) { - options := &PinUpdateSettings{ - Unpin: true, - } + options := &PinUpdateSettings{} for _, opt := range opts { err := opt(options) @@ -64,7 +65,8 @@ func PinUpdateOptions(opts ...PinUpdateOption) (*PinUpdateSettings, error) { type pinType struct{} type pinOpts struct { - Type pinType + Type pinType + Recursively bool } var Pin pinOpts @@ -75,24 +77,42 @@ func (pinType) All() PinLsOption { return Pin.pinType("all") } -// Recursive is an option for Pin.Ls which will make it only return recursive -// pins -func (pinType) Recursive() PinLsOption { - return Pin.pinType("recursive") -} - // Direct is an option for Pin.Ls which will make it only return direct (non // recursive) pins func (pinType) Direct() PinLsOption { return Pin.pinType("direct") } +// Direct is an option for Pin.Ls which will make it only return direct (non +// recursive) pins +func (pinType) Recursive() PinLsOption { + return Pin.pinType("recursive") +} + // Indirect is an option for Pin.Ls which will make it only return indirect pins // (objects referenced by other recursively pinned objects) func (pinType) Indirect() PinLsOption { return Pin.pinType("indirect") } +// Recursive is an option for Pin.Ls which allows to specify whether +// argument should be recursively searched for pins +func (pinOpts) RecursiveList(recursive bool) PinLsOption { + return func(settings *PinLsSettings) error { + settings.Recursive = recursive + return nil + } +} + +// PinPath is an option for Pin.Add which specifies pin path +// default: "default/" +func (pinOpts) PinPath(pinPath string) PinAddOption { + return func(settings *PinAddSettings) error { + settings.PinPath = pinPath + return nil + } +} + // Recursive is an option for Pin.Add which specifies whether to pin an entire // object tree or just one object. Default: true func (pinOpts) Recursive(recursive bool) PinAddOption { @@ -117,12 +137,3 @@ func (pinOpts) pinType(t string) PinLsOption { return nil } } - -// Unpin is an option for Pin.Update which specifies whether to remove the old pin. -// Default is true. -func (pinOpts) Unpin(unpin bool) PinUpdateOption { - return func(settings *PinUpdateSettings) error { - settings.Unpin = unpin - return nil - } -} diff --git a/core/coreapi/interface/pin.go b/core/coreapi/interface/pin.go index 2e119cbeae8b..9152050a4d63 100644 --- a/core/coreapi/interface/pin.go +++ b/core/coreapi/interface/pin.go @@ -11,6 +11,9 @@ type Pin interface { // Path to the pinned object Path() ResolvedPath + // Pinpath of pinned object + PinPath() string + // Type of the pin Type() string } @@ -20,6 +23,9 @@ type PinStatus interface { // Ok indicates whether the pin has been verified to be correct Ok() bool + // Pinpath of pinned object + PinPath() string + // BadNodes returns any bad (usually missing) nodes from the pin BadNodes() []BadPinNode } @@ -37,17 +43,17 @@ type BadPinNode interface { type PinAPI interface { // Add creates new pin, be default recursive - pinning the whole referenced // tree - Add(context.Context, Path, ...options.PinAddOption) error + Add(context.Context, string, Path, ...options.PinAddOption) error // Ls returns list of pinned objects on this node - Ls(context.Context, ...options.PinLsOption) ([]Pin, error) + Ls(context.Context, string, ...options.PinLsOption) ([]Pin, error) // Rm removes pin for object specified by the path - Rm(context.Context, Path) error + Rm(context.Context, string) error // Update changes one pin to another, skipping checks for matching paths in // the old tree - Update(ctx context.Context, from Path, to Path, opts ...options.PinUpdateOption) error + Update(ctx context.Context, from string, to Path, opts ...options.PinUpdateOption) error // Verify verifies the integrity of pinned objects Verify(context.Context) (<-chan PinStatus, error) diff --git a/core/coreapi/object.go b/core/coreapi/object.go index dc237fda01ef..ba349c9869a4 100644 --- a/core/coreapi/object.go +++ b/core/coreapi/object.go @@ -14,7 +14,6 @@ import ( coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/dagutils" - "github.com/ipfs/go-ipfs/pin" dag "gx/ipfs/QmRDaC5z6yXkXTTSWzaxs2sSVBon5RRCN6eNtMmpuHtKCr/go-merkledag" ft "gx/ipfs/QmVNEJ5Vk1e2G5kHMiuVbpD6VQZiK1oS6aWZKjcUQW7hEy/go-unixfs" @@ -127,8 +126,7 @@ func (api *ObjectAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Obj } if options.Pin { - api.node.Pinning.PinWithMode(dagnode.Cid(), pin.Recursive) - err = api.node.Pinning.Flush() + err = api.node.Pinning.AddPin(options.PinPath, dagnode.Cid(), true) if err != nil { return nil, err } diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index d628d29070ff..c87407b1f901 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -10,26 +10,27 @@ import ( merkledag "gx/ipfs/QmRDaC5z6yXkXTTSWzaxs2sSVBon5RRCN6eNtMmpuHtKCr/go-merkledag" bserv "gx/ipfs/QmdHqV7L4bpmMtEXVCrgn8RN6CXqMr3aUeogSkXbJGRtwk/go-blockservice" + "github.com/ipfs/go-ipfs/pin" cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid" offline "gx/ipfs/QmZxjqR9Qgompju73kakSoUj3rbVndAzky3oCDiBNCxPs1/go-ipfs-exchange-offline" ) type PinAPI CoreAPI -func (api *PinAPI) Add(ctx context.Context, p coreiface.Path, opts ...caopts.PinAddOption) error { +func (api *PinAPI) Add(ctx context.Context, pinpath string, p coreiface.Path, opts ...caopts.PinAddOption) error { + settings, err := caopts.PinAddOptions(opts...) if err != nil { return err } - defer api.node.Blockstore.PinLock().Unlock() - rp, err := api.core().ResolvePath(ctx, p) + if err != nil { return err } - _, err = corerepo.Pin(api.node, ctx, []string{rp.Cid().String()}, settings.Recursive) + _, err = corerepo.Pin(api.node, ctx, pinpath, []string{rp.String()}, settings.Recursive) if err != nil { return err } @@ -37,23 +38,23 @@ func (api *PinAPI) Add(ctx context.Context, p coreiface.Path, opts ...caopts.Pin return nil } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]coreiface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, prefix string, opts ...caopts.PinLsOption) ([]coreiface.Pin, error) { settings, err := caopts.PinLsOptions(opts...) if err != nil { return nil, err } switch settings.Type { - case "all", "direct", "indirect", "recursive": + case "all", "direct", "recursive", "indirect": default: return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) } - return api.pinLsAll(settings.Type, ctx) + return api.pinLsAll(settings.Type, prefix, ctx, settings.Recursive) } -func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path) error { - _, err := corerepo.Unpin(api.node, ctx, []string{p.String()}, true, true) +func (api *PinAPI) Rm(ctx context.Context, p string) error { + _, err := corerepo.Unpin(api.node, ctx, []string{p}, true) if err != nil { return err } @@ -61,28 +62,17 @@ func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path) error { return nil } -func (api *PinAPI) Update(ctx context.Context, from coreiface.Path, to coreiface.Path, opts ...caopts.PinUpdateOption) error { - settings, err := caopts.PinUpdateOptions(opts...) - if err != nil { - return err - } - - fp, err := api.core().ResolvePath(ctx, from) - if err != nil { - return err - } - +func (api *PinAPI) Update(ctx context.Context, from string, to coreiface.Path, _ ...caopts.PinUpdateOption) error { tp, err := api.core().ResolvePath(ctx, to) if err != nil { return err } - - return api.node.Pinning.Update(ctx, fp.Cid(), tp.Cid(), settings.Unpin) + return api.node.Pinning.Update(ctx, from, tp.Cid()) } type pinStatus struct { - cid *cid.Cid ok bool + pinPath string badNodes []coreiface.BadPinNode } @@ -96,6 +86,10 @@ func (s *pinStatus) Ok() bool { return s.ok } +func (s *pinStatus) PinPath() string { + return s.pinPath +} + func (s *pinStatus) BadNodes() []coreiface.BadPinNode { return s.badNodes } @@ -113,10 +107,14 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro bs := api.node.Blocks.Blockstore() DAG := merkledag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) getLinks := merkledag.GetLinksWithDAG(DAG) - recPins := api.node.Pinning.RecursiveKeys() + recPins, err := api.node.Pinning.PrefixedPins("", true) + + if err != nil { + return nil, err + } - var checkPin func(root *cid.Cid) *pinStatus - checkPin = func(root *cid.Cid) *pinStatus { + var checkPin func(root *cid.Cid, pinPath string) *pinStatus + checkPin = func(root *cid.Cid, pinPath string) *pinStatus { key := root.String() if status, ok := visited[key]; ok { return status @@ -124,15 +122,15 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro links, err := getLinks(ctx, root) if err != nil { - status := &pinStatus{ok: false, cid: root} + status := &pinStatus{ok: false, pinPath: pinPath} status.badNodes = []coreiface.BadPinNode{&badNode{path: coreiface.IpldPath(root), err: err}} visited[key] = status return status } - status := &pinStatus{ok: true, cid: root} + status := &pinStatus{ok: true} for _, lnk := range links { - res := checkPin(lnk.Cid) + res := checkPin(lnk.Cid, pinPath) if !res.ok { status.ok = false status.badNodes = append(status.badNodes, res.badNodes...) @@ -146,8 +144,8 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro out := make(chan coreiface.PinStatus) go func() { defer close(out) - for _, c := range recPins { - out <- checkPin(c) + for path, c := range recPins { + out <- checkPin(c, path) } }() @@ -156,9 +154,14 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro type pinInfo struct { pinType string + pinPath string path coreiface.ResolvedPath } +func (p *pinInfo) PinPath() string { + return p.pinPath +} + func (p *pinInfo) Path() coreiface.ResolvedPath { return p.path } @@ -167,42 +170,75 @@ func (p *pinInfo) Type() string { return p.pinType } -func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pin, error) { +func (api *PinAPI) pinLsAll(typeStr string, prefix string, ctx context.Context, recursive bool) ([]coreiface.Pin, error) { + keys := make([]coreiface.Pin, 0) - keys := make(map[string]*pinInfo) + var recursiveMap map[string]*cid.Cid + var directMap map[string]*cid.Cid - AddToResultKeys := func(keyList []*cid.Cid, typeStr string) { - for _, c := range keyList { - keys[c.String()] = &pinInfo{ - pinType: typeStr, - path: coreiface.IpldPath(c), - } + if recursive { + var err error + recursiveMap, err = api.node.Pinning.PrefixedPins(prefix, true) + if err != nil { + return nil, err + } + directMap, err = api.node.Pinning.PrefixedPins(prefix, false) + if err != nil { + return nil, err + } + } else { + recursiveMap = make(map[string]*cid.Cid) + c, err := api.node.Pinning.GetPin(prefix, true) + if err != pin.ErrNotPinned && err != nil { + return nil, err + } else if err != pin.ErrNotPinned { + recursiveMap[prefix] = c + } + directMap = make(map[string]*cid.Cid) + c, err = api.node.Pinning.GetPin(prefix, false) + if err != pin.ErrNotPinned && err != nil { + return nil, err + } else if err != pin.ErrNotPinned { + directMap[prefix] = c } } + if typeStr == "recursive" || typeStr == "all" { + for pinPath, c := range recursiveMap { + keys = append(keys, &pinInfo{ + pinType: "recursive", + path: coreiface.IpldPath(c), + pinPath: pinPath, + }) + } + } if typeStr == "direct" || typeStr == "all" { - AddToResultKeys(api.node.Pinning.DirectKeys(), "direct") + for pinPath, c := range directMap { + keys = append(keys, &pinInfo{ + pinType: "direct", + path: coreiface.IpldPath(c), + pinPath: pinPath, + }) + } } if typeStr == "indirect" || typeStr == "all" { - set := cid.NewSet() - for _, k := range api.node.Pinning.RecursiveKeys() { - err := merkledag.EnumerateChildren(ctx, merkledag.GetLinksWithDAG(api.node.DAG), k, set.Visit) + for pinPath, c := range recursiveMap { + set := cid.NewSet() + err := merkledag.EnumerateChildren(ctx, merkledag.GetLinksWithDAG(api.node.DAG), c, set.Visit) if err != nil { return nil, err } + for _, k := range set.Keys() { + keys = append(keys, &pinInfo{ + pinType: "indirect", + pinPath: pinPath, + path: coreiface.IpldPath(k), + }) + } } - AddToResultKeys(set.Keys(), "indirect") - } - if typeStr == "recursive" || typeStr == "all" { - AddToResultKeys(api.node.Pinning.RecursiveKeys(), "recursive") - } - - out := make([]coreiface.Pin, 0, len(keys)) - for _, v := range keys { - out = append(out, v) } - return out, nil + return keys, nil } func (api *PinAPI) core() coreiface.CoreAPI { diff --git a/core/coreapi/pin_test.go b/core/coreapi/pin_test.go index 9bbf16c9cfa2..44feaf2c5a2b 100644 --- a/core/coreapi/pin_test.go +++ b/core/coreapi/pin_test.go @@ -8,6 +8,8 @@ import ( opt "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) +var test_prefix = "test/" + func TestPinAdd(t *testing.T) { ctx := context.Background() _, api, err := makeAPI(ctx) @@ -20,7 +22,7 @@ func TestPinAdd(t *testing.T) { t.Error(err) } - err = api.Pin().Add(ctx, p) + err = api.Pin().Add(ctx, test_prefix, p) if err != nil { t.Error(err) } @@ -38,12 +40,12 @@ func TestPinSimple(t *testing.T) { t.Error(err) } - err = api.Pin().Add(ctx, p) + err = api.Pin().Add(ctx, test_prefix, p) if err != nil { t.Error(err) } - list, err := api.Pin().Ls(ctx) + list, err := api.Pin().Ls(ctx, "test/", opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -60,12 +62,12 @@ func TestPinSimple(t *testing.T) { t.Error("unexpected pin type") } - err = api.Pin().Rm(ctx, p) + err = api.Pin().Rm(ctx, test_prefix+p.Cid().String()) if err != nil { t.Fatal(err) } - list, err = api.Pin().Ls(ctx) + list, err = api.Pin().Ls(ctx, test_prefix, opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -102,17 +104,17 @@ func TestPinRecursive(t *testing.T) { t.Error(err) } - err = api.Pin().Add(ctx, p2) + err = api.Pin().Add(ctx, test_prefix, p2) if err != nil { t.Error(err) } - err = api.Pin().Add(ctx, p3, opt.Pin.Recursive(false)) + err = api.Pin().Add(ctx, test_prefix, p3, opt.Pin.Recursive(false)) if err != nil { t.Error(err) } - list, err := api.Pin().Ls(ctx) + list, err := api.Pin().Ls(ctx, test_prefix, opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -121,7 +123,7 @@ func TestPinRecursive(t *testing.T) { t.Errorf("unexpected pin list len: %d", len(list)) } - list, err = api.Pin().Ls(ctx, opt.Pin.Type.Direct()) + list, err = api.Pin().Ls(ctx, test_prefix, opt.Pin.Type.Direct(), opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -134,7 +136,7 @@ func TestPinRecursive(t *testing.T) { t.Error("unexpected path") } - list, err = api.Pin().Ls(ctx, opt.Pin.Type.Recursive()) + list, err = api.Pin().Ls(ctx, test_prefix, opt.Pin.Type.Recursive(), opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } @@ -147,7 +149,7 @@ func TestPinRecursive(t *testing.T) { t.Error("unexpected path") } - list, err = api.Pin().Ls(ctx, opt.Pin.Type.Indirect()) + list, err = api.Pin().Ls(ctx, test_prefix, opt.Pin.Type.Indirect(), opt.Pin.RecursiveList(true)) if err != nil { t.Fatal(err) } diff --git a/core/corerepo/pinning.go b/core/corerepo/pinning.go index 6c82ad3173cb..87614b42fa16 100644 --- a/core/corerepo/pinning.go +++ b/core/corerepo/pinning.go @@ -25,7 +25,7 @@ import ( cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid" ) -func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]*cid.Cid, error) { +func Pin(n *core.IpfsNode, ctx context.Context, pinpath string, paths []string, recursive bool) ([]*cid.Cid, error) { out := make([]*cid.Cid, len(paths)) r := &resolver.Resolver{ @@ -43,22 +43,17 @@ func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) if err != nil { return nil, fmt.Errorf("pin: %s", err) } - err = n.Pinning.Pin(ctx, dagnode, recursive) + err = n.Pinning.Pin(ctx, pinpath, dagnode, recursive) if err != nil { return nil, fmt.Errorf("pin: %s", err) } out[i] = dagnode.Cid() } - err := n.Pinning.Flush() - if err != nil { - return nil, err - } - return out, nil } -func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool, explain bool) ([]*cid.Cid, error) { +func UnpinPaths(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]*cid.Cid, error) { unpinned := make([]*cid.Cid, len(paths)) r := &resolver.Resolver{ @@ -77,16 +72,26 @@ func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool return nil, err } - err = n.Pinning.Unpin(ctx, k, recursive, explain) + err = n.Pinning.UnpinCid(k, recursive) if err != nil { return nil, err } unpinned[i] = k } - err := n.Pinning.Flush() - if err != nil { - return nil, err + return unpinned, nil +} + +func Unpin(n *core.IpfsNode, ctx context.Context, pinPaths []string, recursive bool) ([]string, error) { + unpinned := make([]string, len(pinPaths)) + + for i, p := range pinPaths { + err := n.Pinning.Unpin(p, recursive) + if err != nil { + return nil, err + } + unpinned[i] = p } + return unpinned, nil } diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 90f97f9b99a0..5448de4b4524 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -79,6 +79,7 @@ type Adder struct { Progress bool Hidden bool Pin bool + PinPath string Trickle bool RawLeaves bool Silent bool @@ -183,15 +184,14 @@ func (adder *Adder) PinRoot() error { } if adder.tempRoot != nil { - err := adder.pinning.Unpin(adder.ctx, adder.tempRoot, true, false) + err := adder.pinning.UnpinCidUnderPrefix("tmp/temproot/", adder.tempRoot, true) if err != nil { return err } adder.tempRoot = rnk } - adder.pinning.PinWithMode(rnk, pin.Recursive) - return adder.pinning.Flush() + return adder.pinning.AddPin(adder.PinPath, rnk, true) } // Finalize flushes the mfs root directory and returns the mfs root node. diff --git a/exchange/reprovide/providers.go b/exchange/reprovide/providers.go index 96b651fd5b32..d3edf90ef2b4 100644 --- a/exchange/reprovide/providers.go +++ b/exchange/reprovide/providers.go @@ -46,14 +46,24 @@ func NewPinnedProvider(pinning pin.Pinner, dag ipld.DAGService, onlyRoots bool) func pinSet(ctx context.Context, pinning pin.Pinner, dag ipld.DAGService, onlyRoots bool) (*streamingSet, error) { set := newStreamingSet() + recursiveKeys, err := pinning.PinnedCids(true) + if err != nil { + return nil, err + } + + directKeys, err := pinning.PinnedCids(false) + if err != nil { + return nil, err + } + go func() { defer close(set.new) - for _, key := range pinning.DirectKeys() { + for _, key := range directKeys { set.add(key) } - for _, key := range pinning.RecursiveKeys() { + for _, key := range recursiveKeys { set.add(key) if !onlyRoots { diff --git a/fuse/ipns/common.go b/fuse/ipns/common.go index d8d8f8c9d493..a21f05a8a6a4 100644 --- a/fuse/ipns/common.go +++ b/fuse/ipns/common.go @@ -18,16 +18,6 @@ func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error { emptyDir := ft.EmptyDirNode() - err := n.Pinning.Pin(ctx, emptyDir, false) - if err != nil { - return err - } - - err = n.Pinning.Flush() - if err != nil { - return err - } - pub := nsys.NewIpnsPublisher(n.Routing, n.Repo.Datastore()) return pub.Publish(ctx, key, path.FromCid(emptyDir.Cid())) diff --git a/namesys/publisher.go b/namesys/publisher.go index 9acab73907b6..69d8b60d886f 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -291,18 +291,6 @@ func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec func InitializeKeyspace(ctx context.Context, pub Publisher, pins pin.Pinner, key ci.PrivKey) error { emptyDir := ft.EmptyDirNode() - // pin recursively because this might already be pinned - // and doing a direct pin would throw an error in that case - err := pins.Pin(ctx, emptyDir, true) - if err != nil { - return err - } - - err = pins.Flush() - if err != nil { - return err - } - return pub.Publish(ctx, key, path.FromCid(emptyDir.Cid())) } diff --git a/pin/gc/gc.go b/pin/gc/gc.go index 672b6a5560c3..64e648f1579c 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -181,10 +181,16 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo } return links, nil } - err := Descendants(ctx, getLinks, gcs, pn.RecursiveKeys()) + recursiveKeys, err := pn.PinnedCids(true) if err != nil { errors = true output <- Result{Error: err} + } else { + err := Descendants(ctx, getLinks, gcs, recursiveKeys) + if err != nil { + errors = true + output <- Result{Error: err} + } } bestEffortGetLinks := func(ctx context.Context, cid *cid.Cid) ([]*ipld.Link, error) { @@ -201,15 +207,14 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo output <- Result{Error: err} } - for _, k := range pn.DirectKeys() { - gcs.Add(k) - } - - err = Descendants(ctx, getLinks, gcs, pn.InternalPins()) + directKeys, err := pn.PinnedCids(false) if err != nil { errors = true output <- Result{Error: err} } + for _, k := range directKeys { + gcs.Add(k) + } if errors { return nil, ErrCannotFetchAllLinks diff --git a/pin/pin.go b/pin/pin.go index edd3caad0f8c..b3e13c125d5f 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -5,43 +5,20 @@ package pin import ( "context" "fmt" - "os" - "sync" - "time" - "github.com/ipfs/go-ipfs/dagutils" + dutils "github.com/ipfs/go-ipfs/dagutils" mdag "gx/ipfs/QmRDaC5z6yXkXTTSWzaxs2sSVBon5RRCN6eNtMmpuHtKCr/go-merkledag" logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" ds "gx/ipfs/QmVG5gxteQNEMhrS8prJSmU2C9rebtFuTd3SYZ5kE3YZ5k/go-datastore" + dsq "gx/ipfs/QmVG5gxteQNEMhrS8prJSmU2C9rebtFuTd3SYZ5kE3YZ5k/go-datastore/query" ipld "gx/ipfs/QmX5CsuHyVZeTLxgRSYkgLSDQKb9UjE8xnhQzCEJWWWFsC/go-ipld-format" cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid" ) var log = logging.Logger("pin") -var pinDatastoreKey = ds.NewKey("/local/pins") - -var emptyKey *cid.Cid - -func init() { - e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") - if err != nil { - log.Error("failed to decode empty key constant") - os.Exit(1) - } - emptyKey = e -} - -const ( - linkRecursive = "recursive" - linkDirect = "direct" - linkIndirect = "indirect" - linkInternal = "internal" - linkNotPinned = "not pinned" - linkAny = "any" - linkAll = "all" -) +var pinDatastoreKey = "/local/pins" // Mode allows to specify different types of pin (recursive, direct etc.). // See the Pin Modes constants for a full list. @@ -58,46 +35,10 @@ const ( // Indirect pins are cids who have some ancestor pinned recursively. Indirect - // Internal pins are cids used to keep the internal state of the pinner. - Internal - // NotPinned NotPinned - - // Any refers to any pinned cid - Any ) -// ModeToString returns a human-readable name for the Mode. -func ModeToString(mode Mode) (string, bool) { - m := map[Mode]string{ - Recursive: linkRecursive, - Direct: linkDirect, - Indirect: linkIndirect, - Internal: linkInternal, - NotPinned: linkNotPinned, - Any: linkAny, - } - s, ok := m[mode] - return s, ok -} - -// StringToMode parses the result of ModeToString() back to a Mode. -// It returns a boolean which is set to false if the mode is unknown. -func StringToMode(s string) (Mode, bool) { - m := map[string]Mode{ - linkRecursive: Recursive, - linkDirect: Direct, - linkIndirect: Indirect, - linkInternal: Internal, - linkNotPinned: NotPinned, - linkAny: Any, - linkAll: Any, // "all" and "any" means the same thing - } - mode, ok := m[s] - return mode, ok -} - // A Pinner provides the necessary methods to keep track of Nodes which are // to be kept locally, according to a pin mode. In practice, a Pinner is in // in charge of keeping the list of items from the local storage that should @@ -107,48 +48,41 @@ type Pinner interface { // and an explanation of why its pinned IsPinned(*cid.Cid) (string, bool, error) - // IsPinnedWithType returns whether or not the given cid is pinned with the - // given pin type, as well as returning the type of pin its pinned with. - IsPinnedWithType(*cid.Cid, Mode) (string, bool, error) + // GetPin returns cid pinned at given path + GetPin(path string, recursive bool) (*cid.Cid, error) // Pin the given node, optionally recursively. - Pin(ctx context.Context, node ipld.Node, recursive bool) error + Pin(ctx context.Context, path string, node ipld.Node, recursive bool) error + + // Unpin the given path with given mode. + Unpin(path string, recursive bool) error - // Unpin the given cid. If recursive is true, removes either a recursive or - // a direct pin. If recursive is false, only removes a direct pin. - Unpin(ctx context.Context, cid *cid.Cid, recursive bool, explain bool) error + // Unpin the given cid with given mode. + UnpinCid(cid *cid.Cid, recursive bool) error + + UnpinCidUnderPrefix(path string, c *cid.Cid, recursive bool) error // Update updates a recursive pin from one cid to another // this is more efficient than simply pinning the new one and unpinning the // old one - Update(ctx context.Context, from, to *cid.Cid, unpin bool) error + Update(ctx context.Context, path string, to *cid.Cid) error + + // Check if given cid is pinned with given mode, return pin path + IsPinPresent(cid *cid.Cid, recursive bool) (string, error) // Check if a set of keys are pinned, more efficient than // calling IsPinned for each key CheckIfPinned(cids ...*cid.Cid) ([]Pinned, error) - // PinWithMode is for manually editing the pin structure. Use with + // AddPin is for manually editing the pin structure. Use with // care! If used improperly, garbage collection may not be // successful. - PinWithMode(*cid.Cid, Mode) - - // RemovePinWithMode is for manually editing the pin structure. - // Use with care! If used improperly, garbage collection may not - // be successful. - RemovePinWithMode(*cid.Cid, Mode) - - // Flush writes the pin state to the backing datastore - Flush() error - - // DirectKeys returns all directly pinned cids - DirectKeys() []*cid.Cid + AddPin(path string, cid *cid.Cid, recursive bool) error - // DirectKeys returns all recursively pinned cids - RecursiveKeys() []*cid.Cid + // PinnedCids returns all pinned cids (recursive or direct) + PinnedCids(recursive bool) ([]*cid.Cid, error) - // InternalPins returns all cids kept pinned for the internal state of the - // pinner - InternalPins() []*cid.Cid + PrefixedPins(prefix string, recursive bool) (map[string]*cid.Cid, error) } // Pinned represents CID which has been pinned with a pinning strategy. @@ -158,6 +92,7 @@ type Pinner interface { type Pinned struct { Key *cid.Cid Mode Mode + Path string Via *cid.Cid } @@ -168,51 +103,43 @@ func (p Pinned) Pinned() bool { // String Returns pin status as string func (p Pinned) String() string { + var result string switch p.Mode { case NotPinned: - return "not pinned" + result = "not pinned" case Indirect: - return fmt.Sprintf("pinned via %s", p.Via) - default: - modeStr, _ := ModeToString(p.Mode) - return fmt.Sprintf("pinned: %s", modeStr) + result = fmt.Sprintf("pinned under %s via %s", p.Path, p.Via) + case Recursive: + result = fmt.Sprintf("pinned under %s recursively", p.Path) + case Direct: + result = fmt.Sprintf("pinned under %s directly", p.Path) } + return result } // pinner implements the Pinner interface type pinner struct { - lock sync.RWMutex - recursePin *cid.Set - directPin *cid.Set - - // Track the keys used for storing the pinning state, so gc does - // not delete them. - internalPin *cid.Set - dserv ipld.DAGService - internal ipld.DAGService // dagservice used to store internal objects - dstore ds.Datastore + dserv ipld.DAGService + dstore ds.Datastore } -// NewPinner creates a new pinner using the given datastore as a backend -func NewPinner(dstore ds.Datastore, serv, internal ipld.DAGService) Pinner { - - rcset := cid.NewSet() - dirset := cid.NewSet() +func NewPinner(dserv ipld.DAGService, dstore ds.Datastore) Pinner { + return &pinner{dserv: dserv, dstore: dstore} +} - return &pinner{ - recursePin: rcset, - directPin: dirset, - dserv: serv, - dstore: dstore, - internal: internal, - internalPin: cid.NewSet(), +// Pin the given node, optionally recursive +// If path ends with /, cid will be appended to path. +func (p *pinner) AddPin(path string, c *cid.Cid, recursive bool) error { + if len(path) == 0 || path[len(path)-1] == '/' { + path += c.String() } + return p.dstore.Put(ds.NewKey(pathToDSKey(path, recursive)), c.Bytes()) } // Pin the given node, optionally recursive -func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { - p.lock.Lock() - defer p.lock.Unlock() +// Also fetches its data +// If path ends with /, cid will be appended to path. +func (p *pinner) Pin(ctx context.Context, path string, node ipld.Node, recurse bool) error { err := p.dserv.Add(ctx, node) if err != nil { return err @@ -221,183 +148,134 @@ func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error { c := node.Cid() if recurse { - if p.recursePin.Has(c) { - return nil - } - - if p.directPin.Has(c) { - p.directPin.Remove(c) - } - // fetch entire graph err := mdag.FetchGraph(ctx, c, p.dserv) if err != nil { return err } - - p.recursePin.Add(c) } else { if _, err := p.dserv.Get(ctx, c); err != nil { return err } + } - if p.recursePin.Has(c) { - return fmt.Errorf("%s already pinned recursively", c.String()) - } + return p.AddPin(path, c, recurse) +} + +func (p *pinner) GetPin(path string, recursive bool) (*cid.Cid, error) { + bytes, err := p.dstore.Get(ds.NewKey(pathToDSKey(path, recursive))) + if err == ds.ErrNotFound { + return nil, ErrNotPinned + } else if err != nil { + return nil, err + } - p.directPin.Add(c) + cid, err := cid.Cast(bytes) + if err != nil { + return nil, err } - return nil + return cid, nil } -// ErrNotPinned is returned when trying to unpin items which are not pinned. var ErrNotPinned = fmt.Errorf("not pinned") // Unpin a given key -func (p *pinner) Unpin(ctx context.Context, c *cid.Cid, recursive bool, explain bool) error { - p.lock.Lock() - defer p.lock.Unlock() - - var pinMode Mode - if recursive { - pinMode = Recursive - } else { - pinMode = Direct - } - _, pinned, err := p.isPinnedWithType(c, pinMode) +func (p *pinner) Unpin(path string, recursive bool) error { + _, err := p.GetPin(path, recursive) if err != nil { return err } - if pinned { - if recursive { - p.recursePin.Remove(c) - return nil - } else { - p.directPin.Remove(c) - return nil - } - } else if recursive { - _, pinned, err := p.isPinnedWithType(c, Direct) - if err != nil { - return err - } - if pinned { - p.directPin.Remove(c) - return nil - } - } - - if explain { - reason, pinned, err := p.isPinnedWithType(c, Any) - if err != nil { - return err - } - if !pinned { - return ErrNotPinned - } - switch reason { - case "recursive": - return fmt.Errorf("%s is pinned recursively", c) - default: - return fmt.Errorf("%s is pinned indirectly under %s", c, reason) - } - } else if recursive { - return fmt.Errorf("%s is not pinned recursively or directly", c) - } else { - return fmt.Errorf("%s is not pinned directly", c) - } - -} -func (p *pinner) isInternalPin(c *cid.Cid) bool { - return p.internalPin.Has(c) + return p.dstore.Delete(ds.NewKey(pathToDSKey(path, recursive))) } -// IsPinned returns whether or not the given key is pinned -// and an explanation of why its pinned -func (p *pinner) IsPinned(c *cid.Cid) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(c, Any) -} +// Unpin a given cid pinned under given prefix +// e.g UCUP("/temp/", QmAFobaz... will unpin /temp/QmAFobaz +// If path does not end in a slash, just acts as Unpin() +func (p *pinner) UnpinCidUnderPrefix(path string, c *cid.Cid, recursive bool) error { + if path[len(path)-1] == '/' { + path += c.String() + } + _, err := p.GetPin(path, recursive) + if err != nil { + return err + } -// IsPinnedWithType returns whether or not the given cid is pinned with the -// given pin type, as well as returning the type of pin its pinned with. -func (p *pinner) IsPinnedWithType(c *cid.Cid, mode Mode) (string, bool, error) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.isPinnedWithType(c, mode) + return p.dstore.Delete(ds.NewKey(pathToDSKey(path, recursive))) } -// isPinnedWithType is the implementation of IsPinnedWithType that does not lock. -// intended for use by other pinned methods that already take locks -func (p *pinner) isPinnedWithType(c *cid.Cid, mode Mode) (string, bool, error) { - switch mode { - case Any, Direct, Indirect, Recursive, Internal: - default: - err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}", - mode, Direct, Indirect, Recursive, Internal, Any) - return "", false, err - } - if (mode == Recursive || mode == Any) && p.recursePin.Has(c) { - return linkRecursive, true, nil - } - if mode == Recursive { - return "", false, nil +func pinKeyPrefix(recursive bool) string { + prefix := "direct" + if recursive { + prefix = "recursive" } + return pinDatastoreKey + "/" + prefix + "/" +} +func pathToDSKey(path string, recursive bool) string { + return pinKeyPrefix(recursive) + path +} - if (mode == Direct || mode == Any) && p.directPin.Has(c) { - return linkDirect, true, nil +func (p *pinner) IsPinPresent(cid *cid.Cid, recursive bool) (string, error) { + pinMap, err := p.PrefixedPins("", recursive) + if err != nil { + return "", err } - if mode == Direct { - return "", false, nil + for path, c := range pinMap { + if c == cid { + return path, nil + } } + return "", ErrNotPinned +} - if (mode == Internal || mode == Any) && p.isInternalPin(c) { - return linkInternal, true, nil - } - if mode == Internal { - return "", false, nil +func (p *pinner) IsPinned(cid *cid.Cid) (string, bool, error) { + pinneds, err := p.CheckIfPinned(cid) + if err != nil { + return "", false, err } - - // Default is Indirect - visitedSet := cid.NewSet() - for _, rc := range p.recursePin.Keys() { - has, err := hasChild(p.dserv, rc, c, visitedSet.Visit) - if err != nil { - return "", false, err - } - if has { - return rc.String(), true, nil - } + if len(pinneds) != 1 { + return "", false, fmt.Errorf("CheckIfPinned returned %d results instead of 1", len(pinneds)) } - return "", false, nil + return pinneds[0].String(), pinneds[0].Pinned(), nil } // CheckIfPinned Checks if a set of keys are pinned, more efficient than // calling IsPinned for each key, returns the pinned status of cid(s) func (p *pinner) CheckIfPinned(cids ...*cid.Cid) ([]Pinned, error) { - p.lock.RLock() - defer p.lock.RUnlock() + recursiveMap, err := p.PrefixedPins("", true) + if err != nil { + return nil, err + } + directMap, err := p.PrefixedPins("", false) + if err != nil { + return nil, err + } pinned := make([]Pinned, 0, len(cids)) toCheck := cid.NewSet() - // First check for non-Indirect pins directly for _, c := range cids { - if p.recursePin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Recursive}) - } else if p.directPin.Has(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Direct}) - } else if p.isInternalPin(c) { - pinned = append(pinned, Pinned{Key: c, Mode: Internal}) - } else { - toCheck.Add(c) + toCheck.Visit(c) + } + + if toCheck.Len() == 0 { + return pinned, nil + } + + for path, c := range directMap { + if toCheck.Has(c) { + pinned = append(pinned, + Pinned{Key: c, Mode: Direct, Path: path}) + toCheck.Remove(c) } } + if toCheck.Len() == 0 { + return pinned, nil + } + // Now walk all recursive pins to check for indirect pins - var checkChildren func(*cid.Cid, *cid.Cid) error - checkChildren = func(rk, parentKey *cid.Cid) error { + var checkChildren func(string, *cid.Cid, *cid.Cid) error + checkChildren = func(path string, rk, parentKey *cid.Cid) error { links, err := ipld.GetLinks(context.TODO(), p.dserv, parentKey) if err != nil { return err @@ -407,11 +285,11 @@ func (p *pinner) CheckIfPinned(cids ...*cid.Cid) ([]Pinned, error) { if toCheck.Has(c) { pinned = append(pinned, - Pinned{Key: c, Mode: Indirect, Via: rk}) + Pinned{Key: c, Mode: Indirect, Via: rk, Path: path}) toCheck.Remove(c) } - err := checkChildren(rk, c) + err := checkChildren(path, rk, c) if err != nil { return err } @@ -423,8 +301,14 @@ func (p *pinner) CheckIfPinned(cids ...*cid.Cid) ([]Pinned, error) { return nil } - for _, rk := range p.recursePin.Keys() { - err := checkChildren(rk, rk) + for path, rk := range recursiveMap { + if toCheck.Has(rk) { + pinned = append(pinned, + Pinned{Key: rk, Mode: Recursive, Path: path}) + toCheck.Remove(rk) + } + + err := checkChildren(path, rk, rk) if err != nil { return nil, err } @@ -441,23 +325,6 @@ func (p *pinner) CheckIfPinned(cids ...*cid.Cid) ([]Pinned, error) { return pinned, nil } -// RemovePinWithMode is for manually editing the pin structure. -// Use with care! If used improperly, garbage collection may not -// be successful. -func (p *pinner) RemovePinWithMode(c *cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Direct: - p.directPin.Remove(c) - case Recursive: - p.recursePin.Remove(c) - default: - // programmer error, panic OK - panic("unrecognized pin type") - } -} - func cidSetWithValues(cids []*cid.Cid) *cid.Set { out := cid.NewSet() for _, c := range cids { @@ -466,168 +333,49 @@ func cidSetWithValues(cids []*cid.Cid) *cid.Set { return out } -// LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (Pinner, error) { - p := new(pinner) +// PrefixedPins returns a map containing all pins of given kind under given prefix +func (p *pinner) PrefixedPins(prefix string, recursive bool) (map[string]*cid.Cid, error) { + result, err := p.dstore.Query(dsq.Query{Prefix: pathToDSKey(prefix, recursive)}) - rootKey, err := d.Get(pinDatastoreKey) - if err != nil { - return nil, fmt.Errorf("cannot load pin state: %v", err) - } - rootCid, err := cid.Cast(rootKey) if err != nil { return nil, err } - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) - defer cancel() + pinMap := make(map[string]*cid.Cid) - root, err := internal.Get(ctx, rootCid) - if err != nil { - return nil, fmt.Errorf("cannot find pinning root object: %v", err) - } - - rootpb, ok := root.(*mdag.ProtoNode) - if !ok { - return nil, mdag.ErrNotProtobuf - } - - internalset := cid.NewSet() - internalset.Add(rootCid) - recordInternal := internalset.Add - - { // load recursive set - recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal) + for entry := range result.Next() { + c, err := cid.Cast(entry.Value) if err != nil { - return nil, fmt.Errorf("cannot load recursive pins: %v", err) + return nil, err } - p.recursePin = cidSetWithValues(recurseKeys) - } - { // load direct set - directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal) - if err != nil { - return nil, fmt.Errorf("cannot load direct pins: %v", err) - } - p.directPin = cidSetWithValues(directKeys) + pinMap[entry.Key[len(pinKeyPrefix(recursive)):]] = c } - - p.internalPin = internalset - - // assign services - p.dserv = dserv - p.dstore = d - p.internal = internal - - return p, nil -} - -// DirectKeys returns a slice containing the directly pinned keys -func (p *pinner) DirectKeys() []*cid.Cid { - return p.directPin.Keys() + return pinMap, nil } -// RecursiveKeys returns a slice containing the recursively pinned keys -func (p *pinner) RecursiveKeys() []*cid.Cid { - return p.recursePin.Keys() -} - -// Update updates a recursive pin from one cid to another +// Update updates a pin from one cid to another // this is more efficient than simply pinning the new one and unpinning the // old one -func (p *pinner) Update(ctx context.Context, from, to *cid.Cid, unpin bool) error { - p.lock.Lock() - defer p.lock.Unlock() +func (p *pinner) Update(ctx context.Context, path string, to *cid.Cid) error { - if !p.recursePin.Has(from) { - return fmt.Errorf("'from' cid was not recursively pinned already") - } - - err := dagutils.DiffEnumerate(ctx, p.dserv, from, to) + c, err := p.GetPin(path, true) if err != nil { return err } - p.recursePin.Add(to) - if unpin { - p.recursePin.Remove(from) - } - return nil -} - -// Flush encodes and writes pinner keysets to the datastore -func (p *pinner) Flush() error { - p.lock.Lock() - defer p.lock.Unlock() - - ctx := context.TODO() - - internalset := cid.NewSet() - recordInternal := internalset.Add - - root := &mdag.ProtoNode{} - { - n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkDirect, n); err != nil { - return err - } - } - - { - n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal) - if err != nil { - return err - } - if err := root.AddNodeLink(linkRecursive, n); err != nil { - return err - } - } - - // add the empty node, its referenced by the pin sets but never created - err := p.internal.Add(ctx, new(mdag.ProtoNode)) + err = dutils.DiffEnumerate(ctx, p.dserv, c, to) if err != nil { return err } - err = p.internal.Add(ctx, root) + err = p.Unpin(path, true) if err != nil { return err } - k := root.Cid() + return p.AddPin(path, to, true) - internalset.Add(k) - if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil { - return fmt.Errorf("cannot store pin state: %v", err) - } - p.internalPin = internalset - return nil -} - -// InternalPins returns all cids kept pinned for the internal state of the -// pinner -func (p *pinner) InternalPins() []*cid.Cid { - p.lock.Lock() - defer p.lock.Unlock() - var out []*cid.Cid - out = append(out, p.internalPin.Keys()...) - return out -} - -// PinWithMode allows the user to have fine grained control over pin -// counts -func (p *pinner) PinWithMode(c *cid.Cid, mode Mode) { - p.lock.Lock() - defer p.lock.Unlock() - switch mode { - case Recursive: - p.recursePin.Add(c) - case Direct: - p.directPin.Add(c) - } } // hasChild recursively looks for a Cid among the children of a root Cid. @@ -655,3 +403,23 @@ func hasChild(ng ipld.NodeGetter, root *cid.Cid, child *cid.Cid, visit func(*cid } return false, nil } + +func (p *pinner) PinnedCids(recursive bool) ([]*cid.Cid, error) { + pinMap, err := p.PrefixedPins("", recursive) + if err != nil { + return nil, err + } + var cids []*cid.Cid + for _, v := range pinMap { + cids = append(cids, v) + } + return cids, nil +} + +func (p *pinner) UnpinCid(cid *cid.Cid, recursive bool) error { + path, err := p.IsPinPresent(cid, recursive) + if err != nil { + return err + } + return p.Unpin(path, recursive) +} diff --git a/pin/pin_test.go b/pin/pin_test.go index 660c7556846f..927d086d41b1 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -16,6 +16,8 @@ import ( blockstore "gx/ipfs/QmcmpX42gtDv1fz24kau4wjS9hfwWj5VexWBKgGnWzsyag/go-ipfs-blockstore" ) +var test_prefix = "test/" + var rand = util.NewTimeSeededRand() func randNode() (*mdag.ProtoNode, *cid.Cid) { @@ -58,7 +60,7 @@ func TestPinnerBasic(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) a, ak := randNode() err := dserv.Add(ctx, a) @@ -67,7 +69,7 @@ func TestPinnerBasic(t *testing.T) { } // Pin A{} - err = p.Pin(ctx, a, false) + err = p.Pin(ctx, test_prefix, a, false) if err != nil { t.Fatal(err) } @@ -101,7 +103,7 @@ func TestPinnerBasic(t *testing.T) { bk := b.Cid() // recursively pin B{A,C} - err = p.Pin(ctx, b, true) + err = p.Pin(ctx, test_prefix, b, true) if err != nil { t.Fatal(err) } @@ -128,7 +130,7 @@ func TestPinnerBasic(t *testing.T) { } // Add D{A,C,E} - err = p.Pin(ctx, d, true) + err = p.Pin(ctx, test_prefix, d, true) if err != nil { t.Fatal(err) } @@ -137,20 +139,12 @@ func TestPinnerBasic(t *testing.T) { assertPinned(t, p, dk, "pinned node not found.") // Test recursive unpin - err = p.Unpin(ctx, dk, true, false) - if err != nil { - t.Fatal(err) - } - - err = p.Flush() + err = p.UnpinCidUnderPrefix(test_prefix, dk, true) if err != nil { t.Fatal(err) } - np, err := LoadPinner(dstore, dserv, dserv) - if err != nil { - t.Fatal(err) - } + np := NewPinner(dserv, dstore) // Test directly pinned assertPinned(t, np, ak, "Could not find pinned node!") @@ -184,7 +178,7 @@ func TestIsPinnedLookup(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) aNodes := make([]*mdag.ProtoNode, aBranchLen) aKeys := make([]*cid.Cid, aBranchLen) @@ -207,7 +201,7 @@ func TestIsPinnedLookup(t *testing.T) { } // Pin A5 recursively - if err := p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil { + if err := p.Pin(ctx, test_prefix, aNodes[aBranchLen-1], true); err != nil { t.Fatal(err) } @@ -245,13 +239,13 @@ func TestIsPinnedLookup(t *testing.T) { // Pin C recursively - if err := p.Pin(ctx, c, true); err != nil { + if err := p.Pin(ctx, test_prefix, c, true); err != nil { t.Fatal(err) } // Pin B recursively - if err := p.Pin(ctx, b, true); err != nil { + if err := p.Pin(ctx, test_prefix, b, true); err != nil { t.Fatal(err) } @@ -261,7 +255,7 @@ func TestIsPinnedLookup(t *testing.T) { assertPinned(t, p, bk, "B should be pinned") // Unpin A5 recursively - if err := p.Unpin(ctx, aKeys[5], true, false); err != nil { + if err := p.UnpinCidUnderPrefix(test_prefix, aKeys[5], true); err != nil { t.Fatal(err) } @@ -269,7 +263,7 @@ func TestIsPinnedLookup(t *testing.T) { assertUnpinned(t, p, aKeys[4], "A4 should be unpinned") // Unpin B recursively - if err := p.Unpin(ctx, bk, true, false); err != nil { + if err := p.UnpinCidUnderPrefix(test_prefix, bk, true); err != nil { t.Fatal(err) } assertUnpinned(t, p, bk, "B should be unpinned") @@ -286,7 +280,7 @@ func TestDuplicateSemantics(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) a, _ := randNode() err := dserv.Add(ctx, a) @@ -295,38 +289,22 @@ func TestDuplicateSemantics(t *testing.T) { } // pin is recursively - err = p.Pin(ctx, a, true) + err = p.Pin(ctx, test_prefix, a, true) if err != nil { t.Fatal(err) } - // pinning directly should fail - err = p.Pin(ctx, a, false) - if err == nil { - t.Fatal("expected direct pin to fail") - } - - // pinning recursively again should succeed - err = p.Pin(ctx, a, true) + // pinning directly should succeed + err = p.Pin(ctx, test_prefix, a, false) if err != nil { t.Fatal(err) } -} -func TestFlush(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - bstore := blockstore.NewBlockstore(dstore) - bserv := bs.New(bstore, offline.Exchange(bstore)) - - dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) - _, k := randNode() - - p.PinWithMode(k, Recursive) - if err := p.Flush(); err != nil { + // pinning recursively again should still not fail + err = p.Pin(ctx, test_prefix, a, true) + if err != nil { t.Fatal(err) } - assertPinned(t, p, k, "expected key to still be pinned") } func TestPinRecursiveFail(t *testing.T) { @@ -336,7 +314,7 @@ func TestPinRecursiveFail(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) a, _ := randNode() b, _ := randNode() @@ -349,7 +327,7 @@ func TestPinRecursiveFail(t *testing.T) { mctx, cancel := context.WithTimeout(ctx, time.Millisecond) defer cancel() - err = p.Pin(mctx, a, true) + err = p.Pin(mctx, test_prefix, a, true) if err == nil { t.Fatal("should have failed to pin here") } @@ -367,7 +345,7 @@ func TestPinRecursiveFail(t *testing.T) { // this one is time based... but shouldnt cause any issues mctx, cancel = context.WithTimeout(ctx, time.Second) defer cancel() - err = p.Pin(mctx, a, true) + err = p.Pin(mctx, test_prefix, a, true) if err != nil { t.Fatal(err) } @@ -381,28 +359,28 @@ func TestPinUpdate(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv, dserv) + p := NewPinner(dserv, dstore) n1, c1 := randNode() n2, c2 := randNode() dserv.Add(ctx, n1) dserv.Add(ctx, n2) - if err := p.Pin(ctx, n1, true); err != nil { + if err := p.Pin(ctx, test_prefix, n1, true); err != nil { t.Fatal(err) } - if err := p.Update(ctx, c1, c2, true); err != nil { + if err := p.Update(ctx, test_prefix+c1.String(), c2); err != nil { t.Fatal(err) } assertPinned(t, p, c2, "c2 should be pinned now") assertUnpinned(t, p, c1, "c1 should no longer be pinned") - if err := p.Update(ctx, c2, c1, false); err != nil { + if err := p.Update(ctx, test_prefix+c1.String(), c1); err != nil { t.Fatal(err) } - assertPinned(t, p, c2, "c2 should be pinned still") + assertUnpinned(t, p, c2, "c2 should no longer be pinned") assertPinned(t, p, c1, "c1 should be pinned now") } diff --git a/pin/set.go b/pin/set.go deleted file mode 100644 index 5b8889ee533b..000000000000 --- a/pin/set.go +++ /dev/null @@ -1,297 +0,0 @@ -package pin - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "hash/fnv" - "sort" - - "github.com/ipfs/go-ipfs/pin/internal/pb" - "gx/ipfs/QmRDaC5z6yXkXTTSWzaxs2sSVBon5RRCN6eNtMmpuHtKCr/go-merkledag" - - ipld "gx/ipfs/QmX5CsuHyVZeTLxgRSYkgLSDQKb9UjE8xnhQzCEJWWWFsC/go-ipld-format" - cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid" - "gx/ipfs/QmdxUuburamoF6zF9qjeQC4WYcWGbWuRmdLacMEsW8ioD8/gogo-protobuf/proto" -) - -const ( - // defaultFanout specifies the default number of fan-out links per layer - defaultFanout = 256 - - // maxItems is the maximum number of items that will fit in a single bucket - maxItems = 8192 -) - -func hash(seed uint32, c *cid.Cid) uint32 { - var buf [4]byte - binary.LittleEndian.PutUint32(buf[:], seed) - h := fnv.New32a() - _, _ = h.Write(buf[:]) - _, _ = h.Write(c.Bytes()) - return h.Sum32() -} - -type itemIterator func() (c *cid.Cid, ok bool) - -type keyObserver func(*cid.Cid) - -type sortByHash struct { - links []*ipld.Link -} - -func (s sortByHash) Len() int { - return len(s.links) -} - -func (s sortByHash) Less(a, b int) bool { - return bytes.Compare(s.links[a].Cid.Bytes(), s.links[b].Cid.Bytes()) == -1 -} - -func (s sortByHash) Swap(a, b int) { - s.links[a], s.links[b] = s.links[b], s.links[a] -} - -func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, depth uint32, iter itemIterator, internalKeys keyObserver) (*merkledag.ProtoNode, error) { - links := make([]*ipld.Link, 0, defaultFanout+maxItems) - for i := 0; i < defaultFanout; i++ { - links = append(links, &ipld.Link{Cid: emptyKey}) - } - - // add emptyKey to our set of internal pinset objects - n := &merkledag.ProtoNode{} - n.SetLinks(links) - - internalKeys(emptyKey) - - hdr := &pb.Set{ - Version: 1, - Fanout: defaultFanout, - Seed: depth, - } - if err := writeHdr(n, hdr); err != nil { - return nil, err - } - - if estimatedLen < maxItems { - // it'll probably fit - links := n.Links() - for i := 0; i < maxItems; i++ { - k, ok := iter() - if !ok { - // all done - break - } - - links = append(links, &ipld.Link{Cid: k}) - } - - n.SetLinks(links) - - // sort by hash, also swap item Data - s := sortByHash{ - links: n.Links()[defaultFanout:], - } - sort.Stable(s) - } - - hashed := make([][]*cid.Cid, defaultFanout) - for { - // This loop essentially enumerates every single item in the set - // and maps them all into a set of buckets. Each bucket will be recursively - // turned into its own sub-set, and so on down the chain. Each sub-set - // gets added to the dagservice, and put into its place in a set nodes - // links array. - // - // Previously, the bucket was selected by taking an int32 from the hash of - // the input key + seed. This was erroneous as we would later be assigning - // the created sub-sets into an array of length 256 by the modulus of the - // int32 hash value with 256. This resulted in overwriting existing sub-sets - // and losing pins. The fix (a few lines down from this comment), is to - // map the hash value down to the 8 bit keyspace here while creating the - // buckets. This way, we avoid any overlapping later on. - k, ok := iter() - if !ok { - break - } - h := hash(depth, k) % defaultFanout - hashed[h] = append(hashed[h], k) - } - - for h, items := range hashed { - if len(items) == 0 { - // recursion base case - continue - } - - childIter := getCidListIterator(items) - - // recursively create a pinset from the items for this bucket index - child, err := storeItems(ctx, dag, uint64(len(items)), depth+1, childIter, internalKeys) - if err != nil { - return nil, err - } - - size, err := child.Size() - if err != nil { - return nil, err - } - - err = dag.Add(ctx, child) - if err != nil { - return nil, err - } - childKey := child.Cid() - - internalKeys(childKey) - - // overwrite the 'empty key' in the existing links array - n.Links()[h] = &ipld.Link{ - Cid: childKey, - Size: size, - } - } - return n, nil -} - -func readHdr(n *merkledag.ProtoNode) (*pb.Set, error) { - hdrLenRaw, consumed := binary.Uvarint(n.Data()) - if consumed <= 0 { - return nil, errors.New("invalid Set header length") - } - - pbdata := n.Data()[consumed:] - if hdrLenRaw > uint64(len(pbdata)) { - return nil, errors.New("impossibly large Set header length") - } - // as hdrLenRaw was <= an int, we now know it fits in an int - hdrLen := int(hdrLenRaw) - var hdr pb.Set - if err := proto.Unmarshal(pbdata[:hdrLen], &hdr); err != nil { - return nil, err - } - - if v := hdr.GetVersion(); v != 1 { - return nil, fmt.Errorf("unsupported Set version: %d", v) - } - if uint64(hdr.GetFanout()) > uint64(len(n.Links())) { - return nil, errors.New("impossibly large Fanout") - } - return &hdr, nil -} - -func writeHdr(n *merkledag.ProtoNode, hdr *pb.Set) error { - hdrData, err := proto.Marshal(hdr) - if err != nil { - return err - } - - // make enough space for the length prefix and the marshaled header data - data := make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)) - - // write the uvarint length of the header data - uvarlen := binary.PutUvarint(data, uint64(len(hdrData))) - - // append the actual protobuf data *after* the length value we wrote - data = append(data[:uvarlen], hdrData...) - - n.SetData(data) - return nil -} - -type walkerFunc func(idx int, link *ipld.Link) error - -func walkItems(ctx context.Context, dag ipld.DAGService, n *merkledag.ProtoNode, fn walkerFunc, children keyObserver) error { - hdr, err := readHdr(n) - if err != nil { - return err - } - // readHdr guarantees fanout is a safe value - fanout := hdr.GetFanout() - for i, l := range n.Links()[fanout:] { - if err := fn(i, l); err != nil { - return err - } - } - for _, l := range n.Links()[:fanout] { - c := l.Cid - children(c) - if c.Equals(emptyKey) { - continue - } - subtree, err := l.GetNode(ctx, dag) - if err != nil { - return err - } - - stpb, ok := subtree.(*merkledag.ProtoNode) - if !ok { - return merkledag.ErrNotProtobuf - } - - if err := walkItems(ctx, dag, stpb, fn, children); err != nil { - return err - } - } - return nil -} - -func loadSet(ctx context.Context, dag ipld.DAGService, root *merkledag.ProtoNode, name string, internalKeys keyObserver) ([]*cid.Cid, error) { - l, err := root.GetNodeLink(name) - if err != nil { - return nil, err - } - - lnkc := l.Cid - internalKeys(lnkc) - - n, err := l.GetNode(ctx, dag) - if err != nil { - return nil, err - } - - pbn, ok := n.(*merkledag.ProtoNode) - if !ok { - return nil, merkledag.ErrNotProtobuf - } - - var res []*cid.Cid - walk := func(idx int, link *ipld.Link) error { - res = append(res, link.Cid) - return nil - } - - if err := walkItems(ctx, dag, pbn, walk, internalKeys); err != nil { - return nil, err - } - return res, nil -} - -func getCidListIterator(cids []*cid.Cid) itemIterator { - return func() (c *cid.Cid, ok bool) { - if len(cids) == 0 { - return nil, false - } - - first := cids[0] - cids = cids[1:] - return first, true - } -} - -func storeSet(ctx context.Context, dag ipld.DAGService, cids []*cid.Cid, internalKeys keyObserver) (*merkledag.ProtoNode, error) { - iter := getCidListIterator(cids) - - n, err := storeItems(ctx, dag, uint64(len(cids)), 0, iter, internalKeys) - if err != nil { - return nil, err - } - err = dag.Add(ctx, n) - if err != nil { - return nil, err - } - internalKeys(n.Cid()) - return n, nil -} diff --git a/pin/set_test.go b/pin/set_test.go deleted file mode 100644 index 69724f4cd375..000000000000 --- a/pin/set_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package pin - -import ( - "context" - "encoding/binary" - "testing" - - dag "gx/ipfs/QmRDaC5z6yXkXTTSWzaxs2sSVBon5RRCN6eNtMmpuHtKCr/go-merkledag" - bserv "gx/ipfs/QmdHqV7L4bpmMtEXVCrgn8RN6CXqMr3aUeogSkXbJGRtwk/go-blockservice" - - ds "gx/ipfs/QmVG5gxteQNEMhrS8prJSmU2C9rebtFuTd3SYZ5kE3YZ5k/go-datastore" - dsq "gx/ipfs/QmVG5gxteQNEMhrS8prJSmU2C9rebtFuTd3SYZ5kE3YZ5k/go-datastore/query" - cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid" - offline "gx/ipfs/QmZxjqR9Qgompju73kakSoUj3rbVndAzky3oCDiBNCxPs1/go-ipfs-exchange-offline" - blockstore "gx/ipfs/QmcmpX42gtDv1fz24kau4wjS9hfwWj5VexWBKgGnWzsyag/go-ipfs-blockstore" -) - -func ignoreCids(_ *cid.Cid) {} - -func objCount(d ds.Datastore) int { - q := dsq.Query{KeysOnly: true} - res, err := d.Query(q) - if err != nil { - panic(err) - } - - var count int - for { - _, ok := res.NextSync() - if !ok { - break - } - - count++ - } - return count -} - -func TestSet(t *testing.T) { - dst := ds.NewMapDatastore() - bstore := blockstore.NewBlockstore(dst) - ds := dag.NewDAGService(bserv.New(bstore, offline.Exchange(bstore))) - - // this value triggers the creation of a recursive shard. - // If the recursive sharding is done improperly, this will result in - // an infinite recursion and crash (OOM) - limit := uint32((defaultFanout * maxItems) + 1) - - var inputs []*cid.Cid - buf := make([]byte, 4) - for i := uint32(0); i < limit; i++ { - binary.BigEndian.PutUint32(buf, i) - c := dag.NewRawNode(buf).Cid() - inputs = append(inputs, c) - } - - _, err := storeSet(context.Background(), ds, inputs[:len(inputs)-1], ignoreCids) - if err != nil { - t.Fatal(err) - } - - objs1 := objCount(dst) - - out, err := storeSet(context.Background(), ds, inputs, ignoreCids) - if err != nil { - t.Fatal(err) - } - - objs2 := objCount(dst) - if objs2-objs1 > 2 { - t.Fatal("set sharding does not appear to be deterministic") - } - - // weird wrapper node because loadSet expects us to pass an - // object pointing to multiple named sets - setroot := &dag.ProtoNode{} - err = setroot.AddNodeLink("foo", out) - if err != nil { - t.Fatal(err) - } - - outset, err := loadSet(context.Background(), ds, setroot, "foo", ignoreCids) - if err != nil { - t.Fatal(err) - } - - if uint32(len(outset)) != limit { - t.Fatal("got wrong number", len(outset), limit) - } - - seen := cid.NewSet() - for _, c := range outset { - seen.Add(c) - } - - for _, c := range inputs { - if !seen.Has(c) { - t.Fatalf("expected to have '%s', didnt find it", c) - } - } -} diff --git a/test/sharness/t0010-basic-commands.sh b/test/sharness/t0010-basic-commands.sh index 2dc69933413d..cffcd01191b9 100755 --- a/test/sharness/t0010-basic-commands.sh +++ b/test/sharness/t0010-basic-commands.sh @@ -122,7 +122,7 @@ test_expect_success "'ipfs commands --flags' succeeds" ' ' test_expect_success "'ipfs commands --flags' output looks good" ' - grep "ipfs pin add --recursive / ipfs pin add -r" commands.txt && + grep "ipfs pin add --direct / ipfs pin add -d" commands.txt && grep "ipfs id --format / ipfs id -f" commands.txt && grep "ipfs repo gc --quiet / ipfs repo gc -q" commands.txt ' diff --git a/test/sharness/t0025-datastores.sh b/test/sharness/t0025-datastores.sh index 21100e5ae317..2796c2f69a00 100755 --- a/test/sharness/t0025-datastores.sh +++ b/test/sharness/t0025-datastores.sh @@ -10,7 +10,7 @@ test_expect_success "'ipfs init --profile=badgerds' succeeds" ' ' test_expect_success "'ipfs pin ls' works" ' - ipfs pin ls | wc -l | grep 9 + ipfs pin ls -r | wc -l | grep 8 ' test_done diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index bd60140f58f6..224afed055e0 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -309,7 +309,7 @@ test_add_named_pipe() { err_prefix=$1 test_expect_success "useful error message when adding a named pipe" ' mkfifo named-pipe && - test_expect_code 1 ipfs add named-pipe 2>actual && + test_expect_code 1 ipfs add -P named-pipe named-pipe 2>actual && STAT=$(generic_stat named-pipe) && rm named-pipe && grep "Error: Unrecognized file type for named-pipe: $STAT" actual && @@ -321,7 +321,7 @@ test_add_named_pipe() { mkdir -p named-pipe-dir && mkfifo named-pipe-dir/named-pipe && STAT=$(generic_stat named-pipe-dir/named-pipe) && - test_expect_code 1 ipfs add -r named-pipe-dir 2>actual && + test_expect_code 1 ipfs add -P named-pipe -r named-pipe-dir 2>actual && printf "Error:$err_prefix Unrecognized file type for named-pipe-dir/named-pipe: $STAT\n" >expected && rm named-pipe-dir/named-pipe && rmdir named-pipe-dir && @@ -590,7 +590,7 @@ test_add_cat_expensive "--cid-version=1" "zdj7WcatQrtuE4WMkS4XsfsMixuQN2po4irkYh # encoded with the blake2b-256 hash funtion test_add_cat_expensive '--hash=blake2b-256' "zDMZof1kwndounDzQCANUHjiE3zt1mPEgx7RE3JTHoZrRRa79xcv" -test_add_named_pipe " Post http://$API_ADDR/api/v0/add?chunker=size-262144&encoding=json&hash=sha2-256&inline-limit=32&pin=true&progress=true&recursive=true&stream-channels=true:" +test_add_named_pipe " Post http://$API_ADDR/api/v0/add?chunker=size-262144&encoding=json&hash=sha2-256&pin=true&inline-limit=32&pinpath=named-pipe&progress=true&recursive=true&stream-channels=true:" test_add_pwd_is_symlink diff --git a/test/sharness/t0046-id-hash.sh b/test/sharness/t0046-id-hash.sh index 27e49e855e15..3d49c01b2bc2 100755 --- a/test/sharness/t0046-id-hash.sh +++ b/test/sharness/t0046-id-hash.sh @@ -34,7 +34,7 @@ test_expect_success "but can fetch it anyway" ' ' test_expect_success "block rm does nothing" ' - ipfs pin rm $HASH && + ipfs pin rm added/$HASH && ipfs block rm $HASH ' diff --git a/test/sharness/t0050-block.sh b/test/sharness/t0050-block.sh index 729942d3fc02..4218e84008b3 100755 --- a/test/sharness/t0050-block.sh +++ b/test/sharness/t0050-block.sh @@ -80,7 +80,6 @@ test_expect_success "add and pin directory" ' echo "file2" > adir/file2 && echo "file3" > adir/file3 && ipfs add -r adir - ipfs pin add -r $DIRHASH ' test_expect_success "can't remove pinned block" ' @@ -88,7 +87,7 @@ test_expect_success "can't remove pinned block" ' ' test_expect_success "can't remove pinned block: output looks good" ' - grep -q "$DIRHASH: pinned: recursive" block_rm_err + grep -q "$DIRHASH: pinned under added/$DIRHASH recursively" block_rm_err ' test_expect_success "can't remove indirectly pinned block" ' @@ -96,11 +95,11 @@ test_expect_success "can't remove indirectly pinned block" ' ' test_expect_success "can't remove indirectly pinned block: output looks good" ' - grep -q "$FILE1HASH: pinned via $DIRHASH" block_rm_err + grep -q "$FILE1HASH: pinned under added/$DIRHASH via $DIRHASH" block_rm_err ' test_expect_success "remove pin" ' - ipfs pin rm -r $DIRHASH + ipfs pin rm added/$DIRHASH ' test_expect_success "multi-block 'ipfs block rm' succeeds" ' @@ -120,7 +119,7 @@ test_expect_success "'add some blocks' succeeds" ' test_expect_success "add and pin directory" ' ipfs add -r adir - ipfs pin add -r $DIRHASH + ipfs pin add $DIRHASH ' HASH=QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index 79b102baeed2..53fe34b529d4 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -21,7 +21,7 @@ test_expect_success "'ipfs add afile' succeeds" ' ' test_expect_success "added file was pinned" ' - ipfs pin ls --type=recursive >actual && + ipfs pin ls -r added/ --type=recursive >actual && grep "$HASH" actual ' @@ -40,11 +40,11 @@ test_expect_success "'ipfs repo gc' doesnt remove file" ' ' test_expect_success "'ipfs pin rm' succeeds" ' - ipfs pin rm -r "$HASH" >actual1 + ipfs pin rm "added/$HASH" >actual1 ' test_expect_success "'ipfs pin rm' output looks good" ' - echo "unpinned $HASH" >expected1 && + echo "unpinned added/$HASH" >expected1 && test_cmp expected1 actual1 ' @@ -54,7 +54,7 @@ test_expect_success "ipfs repo gc fully reverse ipfs add" ' expected="$(directory_size "$IPFS_PATH/blocks")" && find "$IPFS_PATH/blocks" -type f && hash=$(ipfs add -q gcfile) && - ipfs pin rm -r $hash && + ipfs pin rm added/$hash && ipfs repo gc && actual=$(directory_size "$IPFS_PATH/blocks") && { test "$actual" -eq "$expected" || test_fsh echo "$actual != $expected"; } && @@ -62,58 +62,36 @@ test_expect_success "ipfs repo gc fully reverse ipfs add" ' ' test_expect_success "file no longer pinned" ' - ipfs pin ls --type=recursive --quiet >actual2 && + ipfs pin ls -r --type=recursive --quiet >actual2 && test_expect_code 1 grep $HASH actual2 ' test_expect_success "recursively pin afile(default action)" ' - HASH=`ipfs add -q afile` && + HASH=`ipfs add -q --pin=false afile` && ipfs pin add "$HASH" ' test_expect_success "recursively pin rm afile (default action)" ' - ipfs pin rm "$HASH" + ipfs pin rm "default/$HASH" ' test_expect_success "recursively pin afile" ' - ipfs pin add -r "$HASH" -' - -test_expect_success "pinning directly should fail now" ' - echo "Error: pin: $HASH already pinned recursively" >expected3 && - test_must_fail ipfs pin add -r=false "$HASH" 2>actual3 && - test_cmp expected3 actual3 -' - -test_expect_success "'ipfs pin rm -r=false ' should fail" ' - echo "Error: $HASH is not pinned directly" >expected4 && - test_must_fail ipfs pin rm -r=false "$HASH" 2>actual4 && - test_cmp expected4 actual4 -' - -test_expect_success "'ipfs pin rm -e -r=false ' should fail and explain why" ' - echo "Error: $HASH is pinned recursively" >expected11 && - test_must_fail ipfs pin rm -e -r=false "$HASH" 2>actual11 && - test_cmp expected11 actual11 + ipfs pin add "$HASH" ' -test_expect_success "remove recursive pin, add direct" ' - echo "unpinned $HASH" >expected5 && - ipfs pin rm -r "$HASH" >actual5 && - test_cmp expected5 actual5 && - ipfs pin add -r=false "$HASH" +test_expect_success "pinning directly should not fail either" ' + ipfs pin add -d "$HASH" ' -test_expect_success "remove direct pin" ' - echo "unpinned $HASH" >expected6 && - ipfs pin rm -r=false "$HASH" >actual6 && +test_expect_success "remove recursive pin" ' + echo "unpinned default/$HASH" >expected6 && + ipfs pin rm "default/$HASH" >actual6 && test_cmp expected6 actual6 ' -test_expect_success "remove direct pin with pin rm -r" ' - echo "unpinned $HASH" >expected6 && - ipfs pin add -r=false "$HASH" - ipfs pin rm "$HASH" >actual6 && +test_expect_success "remove direct pin" ' + echo "unpinned default/$HASH" >expected6 && + ipfs pin rm -d "default/$HASH" >actual6 && test_cmp expected6 actual6 ' @@ -139,37 +117,38 @@ test_expect_success "adding multiblock random file succeeds" ' ' test_expect_success "'ipfs pin ls --type=indirect' is correct" ' - ipfs refs "$MBLOCKHASH" >refsout && - ipfs refs -r "$HASH_WELCOME_DOCS" >>refsout && - sed -i"~" "s/\(.*\)/\1 indirect/g" refsout && - ipfs pin ls --type=indirect >indirectpins && + ipfs refs "$MBLOCKHASH" >mbrefsout && + ipfs refs -r "$HASH_WELCOME_DOCS" >assetsrefsout && + sed -i"~" "s|\(.*\)|\1 indirect added/$MBLOCKHASH|g" mbrefsout && + sed -i"~" "s/\(.*\)/\1 indirect assets/g" assetsrefsout && + ipfs pin ls -r --type=indirect >indirectpins && + cat assetsrefsout mbrefsout > refsout && test_sort_cmp refsout indirectpins ' test_expect_success "pin something directly" ' echo "ipfs is so awesome" >awesome && DIRECTPIN=`ipfs add -q awesome` && - echo "unpinned $DIRECTPIN" >expected9 && - ipfs pin rm -r "$DIRECTPIN" >actual9 && + echo "unpinned added/$DIRECTPIN" >expected9 && + ipfs pin rm "added/$DIRECTPIN" >actual9 && test_cmp expected9 actual9 && echo "pinned $DIRECTPIN directly" >expected10 && - ipfs pin add -r=false "$DIRECTPIN" >actual10 && + ipfs pin add -d "$DIRECTPIN" >actual10 && test_cmp expected10 actual10 ' test_expect_success "'ipfs pin ls --type=direct' is correct" ' - echo "$DIRECTPIN direct" >directpinexpected && - ipfs pin ls --type=direct >directpinout && + echo "$DIRECTPIN direct default/$DIRECTPIN" >directpinexpected && + ipfs pin ls -r --type=direct >directpinout && test_sort_cmp directpinexpected directpinout ' test_expect_success "'ipfs pin ls --type=recursive' is correct" ' - echo "$MBLOCKHASH" >rp_expected && - echo "$HASH_WELCOME_DOCS" >>rp_expected && - echo "$EMPTY_DIR" >>rp_expected && - sed -i"~" "s/\(.*\)/\1 recursive/g" rp_expected && - ipfs pin ls --type=recursive >rp_actual && + echo "$MBLOCKHASH added/$MBLOCKHASH" >rp_expected && + echo "$HASH_WELCOME_DOCS assets" >>rp_expected && + sed -i"~" -E "s/^([^ ]+)/\1 recursive/g" rp_expected && + ipfs pin ls -r --type=recursive >rp_actual && test_sort_cmp rp_expected rp_actual ' @@ -178,7 +157,7 @@ test_expect_success "'ipfs pin ls --type=all --quiet' is correct" ' cat rp_actual >>allpins && cat indirectpins >>allpins && cut -f1 -d " " allpins | sort | uniq >> allpins_uniq_hashes && - ipfs pin ls --type=all --quiet >actual_allpins && + ipfs pin ls -r --type=all --quiet >actual_allpins && test_sort_cmp allpins_uniq_hashes actual_allpins ' diff --git a/test/sharness/t0081-repo-pinning.sh b/test/sharness/t0081-repo-pinning.sh index 54e64253c043..2610fbb4d35c 100755 --- a/test/sharness/t0081-repo-pinning.sh +++ b/test/sharness/t0081-repo-pinning.sh @@ -15,7 +15,7 @@ test_pin_flag() { echo "test_pin_flag" "$@" - if ipfs pin ls --type="$ptype" "$object" >actual + if ipfs pin ls -r --type="$ptype" | grep "^$object" >actual then test "$expect" = "true" && return test_fsh cat actual @@ -141,7 +141,7 @@ test_expect_success "added dir was NOT pinned indirectly" ' ' test_expect_success "nothing is pinned directly" ' - ipfs pin ls --type=direct >actual4 && + ipfs pin ls -r --type=direct >actual4 && test_must_be_empty actual4 ' @@ -166,8 +166,8 @@ test_expect_success "objects are still there" ' ' test_expect_success "remove dir recursive pin succeeds" ' - echo "unpinned $HASH_DIR1" >expected5 && - ipfs pin rm -r "$HASH_DIR1" >actual5 && + echo "unpinned added/$HASH_DIR1" >expected5 && + ipfs pin rm "added/$HASH_DIR1" >actual5 && test_cmp expected5 actual5 ' @@ -178,16 +178,16 @@ test_expect_success "none are pinned any more" ' test_pin "$HASH_FILE3" && test_pin "$HASH_FILE2" && test_pin "$HASH_FILE1" && - test_pin "$HASH_DIR3" && - test_pin "$HASH_DIR4" && - test_pin "$HASH_DIR2" && + test_pin "$HASH_DIR3" && + test_pin "$HASH_DIR4" && + test_pin "$HASH_DIR2" && test_pin "$HASH_DIR1" ' test_expect_success "pin some directly and indirectly" ' - ipfs pin add -r=false "$HASH_DIR1" >actual7 && - ipfs pin add -r=true "$HASH_DIR2" >>actual7 && - ipfs pin add -r=false "$HASH_FILE1" >>actual7 && + ipfs pin add -d "$HASH_DIR1" >actual7 && + ipfs pin add "$HASH_DIR2" >>actual7 && + ipfs pin add -d "$HASH_FILE1" >>actual7 && echo "pinned $HASH_DIR1 directly" >expected7 && echo "pinned $HASH_DIR2 recursively" >>expected7 && echo "pinned $HASH_FILE1 directly" >>expected7 && @@ -195,10 +195,10 @@ test_expect_success "pin some directly and indirectly" ' ' test_expect_success "pin lists look good" ' - test_pin $HASH_DIR1 direct && - test_pin $HASH_DIR2 recursive && - test_pin $HASH_DIR3 && - test_pin $HASH_DIR4 indirect && + test_pin $HASH_DIR1 direct && + test_pin $HASH_DIR2 recursive && + test_pin $HASH_DIR3 && + test_pin $HASH_DIR4 indirect && test_pin $HASH_FILE1 indirect direct && test_pin $HASH_FILE2 indirect && test_pin $HASH_FILE3 && @@ -226,7 +226,7 @@ test_expect_success "some objects are still there" ' ipfs cat "$HASH_FILE1" >>actual8 && ipfs ls "$HASH_DIR4" >>actual8 && ipfs ls "$HASH_DIR2" >>actual8 && - ipfs object links "$HASH_DIR1" >>actual8 && + ipfs object links "$HASH_DIR1" >>actual8 && test_cmp expected8 actual8 ' @@ -238,8 +238,8 @@ test_expect_success "some are no longer there" ' ' test_expect_success "recursive pin fails without objects" ' - ipfs pin rm -r=false "$HASH_DIR1" && - test_must_fail ipfs pin add -r "$HASH_DIR1" 2>err_expected8 && + ipfs pin rm -d "default/$HASH_DIR1" && + test_must_fail ipfs pin add "$HASH_DIR1" 2>err_expected8 && grep "pin: merkledag: not found" err_expected8 || test_fsh cat err_expected8 ' diff --git a/test/sharness/t0085-pins.sh b/test/sharness/t0085-pins.sh index 9fac885e887e..f7888c596073 100755 --- a/test/sharness/t0085-pins.sh +++ b/test/sharness/t0085-pins.sh @@ -46,19 +46,19 @@ test_pins() { ' test_expect_success "unpin those hashes" ' - cat hashes | ipfs pin rm + cat hashes | sed "s|\(.*\)|default/\1|" | ipfs pin rm ' test_expect_success "test pin update" ' - ipfs pin add "$HASH_A" && - ipfs pin ls > before_update && + ipfs pin add -P test/updated_pin "$HASH_A" && + ipfs pin ls -r > before_update && test_should_contain "$HASH_A" before_update && test_must_fail grep -q "$HASH_B" before_update && - ipfs pin update --unpin=true "$HASH_A" "$HASH_B" && - ipfs pin ls > after_update && + ipfs pin update test/updated_pin "$HASH_B" && + ipfs pin ls -r > after_update && test_must_fail grep -q "$HASH_A" after_update && test_should_contain "$HASH_B" after_update && - ipfs pin rm "$HASH_B" + ipfs pin rm test/updated_pin ' } @@ -86,11 +86,11 @@ test_pin_dag() { test_pin_dag_init $1 test_expect_success "'ipfs pin add --progress' file" ' - ipfs pin add --recursive=true $HASH + ipfs pin add $HASH ' test_expect_success "'ipfs pin rm' file" ' - ipfs pin rm $HASH + ipfs pin rm default/$HASH ' test_expect_success "remove part of the dag" ' @@ -99,7 +99,7 @@ test_pin_dag() { ' test_expect_success "pin file, should fail" ' - test_must_fail ipfs pin add --recursive=true $HASH 2> err && + test_must_fail ipfs pin add $HASH 2> err && cat err && grep -q "not found" err ' diff --git a/test/sharness/t0087-repo-robust-gc.sh b/test/sharness/t0087-repo-robust-gc.sh index 896fc6929b44..9057d788bf8e 100755 --- a/test/sharness/t0087-repo-robust-gc.sh +++ b/test/sharness/t0087-repo-robust-gc.sh @@ -13,7 +13,7 @@ test_gc_robust_part1() { test_expect_success "add a 1MB file with --raw-leaves" ' random 1048576 56 > afile && - HASH1=`ipfs add --raw-leaves -q afile` + HASH1=`ipfs add --raw-leaves -q -P test/afile afile` ' HASH1FILE=.ipfs/blocks/L3/CIQNIPL4GP62ZMNNSLZ2G33Z3T5VAN3YHCJTGT5FG45XWH5FGZRXL3A.data @@ -56,7 +56,7 @@ test_gc_robust_part1() { ' test_expect_success "unpin the 1MB file" ' - ipfs pin rm $HASH1 + ipfs pin rm test/afile ' # make sure the permission problem is fixed on exit, otherwise cleanup @@ -91,7 +91,7 @@ test_gc_robust_part2() { test_expect_success "add 1MB file normally (i.e., without raw leaves)" ' random 1048576 56 > afile && - HASH2=`ipfs add -q afile` + HASH2=`ipfs add -q -P test/afile afile` ' LEAF1=QmSijovevteoY63Uj1uC5b8pkpDU5Jgyk2dYBqz3sMJUPc @@ -142,7 +142,7 @@ test_gc_robust_part2() { ' test_expect_success "unpin 1MB file" ' - ipfs pin rm $HASH2 + ipfs pin rm test/afile ' test_expect_success "'ipfs repo gc' should be fine now" ' diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index 7b97f31ded61..3a5b6c7be6f0 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -92,7 +92,7 @@ test_sharding() { test_expect_success "can unpin a file from sharded directory $EXTRA" ' read -r _ HASH _ < pin_hash && - ipfs pin rm $HASH + ipfs pin rm default/$HASH ' test_expect_success "output object was really sharded and has correct hash $EXTRA" ' diff --git a/test/sharness/t0252-files-gc.sh b/test/sharness/t0252-files-gc.sh index 7f513a6214f5..a97eb80d0976 100755 --- a/test/sharness/t0252-files-gc.sh +++ b/test/sharness/t0252-files-gc.sh @@ -29,12 +29,12 @@ test_expect_success "gc okay after adding incomplete node -- prep" ' ipfs files mkdir /adir && echo "file1" | ipfs files write --create /adir/file1 && echo "file2" | ipfs files write --create /adir/file2 && - ipfs pin add --recursive=false $ADIR_HASH && + ipfs pin add -d $ADIR_HASH && ipfs files rm -r /adir && ipfs repo gc && # will remove /adir/file1 and /adir/file2 but not /adir test_must_fail ipfs cat $FILE1_HASH && ipfs files cp /ipfs/$ADIR_HASH /adir && - ipfs pin rm -r=false $ADIR_HASH + ipfs pin rm -d default/$ADIR_HASH ' test_expect_success "gc okay after adding incomplete node" ' @@ -49,7 +49,7 @@ test_expect_success "add directory with direct pin" ' FILE_UNPINNED=$(ipfs add --pin=false -q -r mydir/hello.txt) && DIR_PINNED=$(ipfs add --pin=false -q -r mydir | tail -n1) && ipfs add --pin=false -r mydir && - ipfs pin add --recursive=false $DIR_PINNED && + ipfs pin add -d $DIR_PINNED && ipfs cat $FILE_UNPINNED ' diff --git a/test/sharness/t0260-sharding.sh b/test/sharness/t0260-sharding.sh index 01fc447f43b9..0af5547ea51e 100755 --- a/test/sharness/t0260-sharding.sh +++ b/test/sharness/t0260-sharding.sh @@ -68,7 +68,7 @@ test_expect_success "ipfs cat error output the same" ' test_expect_success "'ipfs ls --resolve-type=false' admits missing block" ' ipfs ls "$SHARDED" | head -1 > first_file && read -r HASH _ NAME /dev/null ' @@ -113,7 +113,7 @@ test_expect_success "files can not be retrieved via the urlstore" ' ' test_expect_success "remove broken files" ' - ipfs pin rm $HASH1 $HASH2 && + ipfs pin rm default/$HASH1 default/$HASH2 && ipfs repo gc > /dev/null ' diff --git a/test/sharness/t0276-cidv0v1.sh b/test/sharness/t0276-cidv0v1.sh index d0b33f077d1d..ebb30660c89c 100755 --- a/test/sharness/t0276-cidv0v1.sh +++ b/test/sharness/t0276-cidv0v1.sh @@ -59,7 +59,7 @@ test_expect_success "make sure the CIDv1 hash is not in the repo" ' ' test_expect_success "clean up" ' - ipfs pin rm $AHASHv0 && + ipfs pin rm added/$AHASHv0 && ipfs repo gc && ! ipfs refs local | grep -q $AHASHv0 ' diff --git a/test/sharness/t0600-issues-and-regressions-online.sh b/test/sharness/t0600-issues-and-regressions-online.sh index 6227968de510..cea8b8622961 100755 --- a/test/sharness/t0600-issues-and-regressions-online.sh +++ b/test/sharness/t0600-issues-and-regressions-online.sh @@ -35,14 +35,14 @@ test_expect_success "metrics work" ' test_expect_success "pin add api looks right - #3753" ' HASH=$(echo "foo" | ipfs add -q) && - curl "http://$API_ADDR/api/v0/pin/add/$HASH" > pinadd_out && + curl "http://$API_ADDR/api/v0/pin/add/$HASH?pinpath=pintest" > pinadd_out && echo "{\"Pins\":[\"QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6\"]}" > pinadd_exp && test_cmp pinadd_out pinadd_exp ' test_expect_success "pin add api looks right - #3753" ' - curl "http://$API_ADDR/api/v0/pin/rm/$HASH" > pinrm_out && - echo "{\"Pins\":[\"QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6\"]}" > pinrm_exp && + curl "http://$API_ADDR/api/v0/pin/rm/pintest" > pinrm_out && + echo "{\"Pins\":[\"pintest\"]}" > pinrm_exp && test_cmp pinrm_out pinrm_exp ' diff --git a/test/sharness/x0601-pin-fail-test.sh b/test/sharness/x0601-pin-fail-test.sh index ffab1062d07c..301bc225f8d2 100755 --- a/test/sharness/x0601-pin-fail-test.sh +++ b/test/sharness/x0601-pin-fail-test.sh @@ -14,7 +14,7 @@ test_launch_ipfs_daemon test_expect_success "pre-test setup" ' printf "" > pins && - ipfs pin ls --type=recursive -q > rec_pins_before + ipfs pin ls -r --type=recursive -q > rec_pins_before ' @@ -26,7 +26,7 @@ do done test_expect_success "get pinset afterwards" ' - ipfs pin ls --type=recursive -q | sort > rec_pins_after && + ipfs pin ls -r --type=recursive -q | sort > rec_pins_after && cat pins rec_pins_before | sort | uniq > exp_pins_after && test_cmp rec_pins_after exp_pins_after '