From 1a835202b23983fb74ded8d365a63219eb427e4f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 6 Oct 2017 14:17:30 +0300 Subject: [PATCH 01/13] filestore: add URLStore License: MIT Signed-off-by: Jakub Sztandera --- core/commands/commands_test.go | 2 + core/commands/root.go | 3 +- core/commands/urlstore.go | 84 ++++++++++++++++++++++++++++++++++ core/coreunix/add.go | 2 +- core/coreunix/add_test.go | 2 +- filestore/filestore.go | 2 +- filestore/filestore_test.go | 2 +- filestore/fsrefstore.go | 76 +++++++++++++++++++++++++----- filestore/pb/Makefile | 12 ++--- filestore/pb/dataobj.pb.go | 12 ++++- filestore/pb/dataobj.proto | 1 + importer/helpers/dagbuilder.go | 10 +++- importer/helpers/helpers.go | 5 +- package.json | 4 +- 14 files changed, 186 insertions(+), 31 deletions(-) create mode 100644 core/commands/urlstore.go diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 91c05c36dc3..2c9788cc452 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -210,6 +210,8 @@ func TestCommands(t *testing.T) { "/tar/add", "/tar/cat", "/update", + "/urlstore", + "/urlstore/add", "/version", } diff --git a/core/commands/root.go b/core/commands/root.go index 9aa8271613b..1f62b720520 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -5,12 +5,12 @@ import ( "strings" oldcmds "github.com/ipfs/go-ipfs/commands" + lgc "github.com/ipfs/go-ipfs/commands/legacy" dag "github.com/ipfs/go-ipfs/core/commands/dag" e "github.com/ipfs/go-ipfs/core/commands/e" ocmd "github.com/ipfs/go-ipfs/core/commands/object" unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs" - lgc "github.com/ipfs/go-ipfs/commands/legacy" "gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" @@ -136,6 +136,7 @@ var rootSubcommands = map[string]*cmds.Command{ "tar": lgc.NewCommand(TarCmd), "file": lgc.NewCommand(unixfs.UnixFSCmd), "update": lgc.NewCommand(ExternalBinary()), + "urlstore": lgc.NewCommand(UrlStoreCmd), "version": lgc.NewCommand(VersionCmd), "shutdown": lgc.NewCommand(daemonShutdownCmd), } diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go new file mode 100644 index 00000000000..03647e7a7c2 --- /dev/null +++ b/core/commands/urlstore.go @@ -0,0 +1,84 @@ +package commands + +import ( + "fmt" + "io" + "net/http" + "strings" + + cmds "github.com/ipfs/go-ipfs/commands" + balanced "github.com/ipfs/go-ipfs/importer/balanced" + ihelper "github.com/ipfs/go-ipfs/importer/helpers" + + mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" + chunk "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker" + cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" + cmdkit "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" +) + +var UrlStoreCmd = &cmds.Command{ + + Subcommands: map[string]*cmds.Command{ + "add": urlAdd, + }, +} + +var urlAdd = &cmds.Command{ + Arguments: []cmdkit.Argument{ + cmdkit.StringArg("url", true, false, "URL to add to IPFS"), + }, + Type: BlockStat{}, + + Run: func(req cmds.Request, res cmds.Response) { + url := req.Arguments()[0] + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + hreq, err := http.NewRequest("GET", url, nil) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + hres, err := http.DefaultClient.Do(hreq) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + if hres.StatusCode != http.StatusOK { + res.SetError(fmt.Errorf("expected code 200, got: %d", hres.StatusCode), cmdkit.ErrNormal) + return + } + + chk := chunk.NewSizeSplitter(hres.Body, chunk.DefaultBlockSize) + prefix := cid.NewPrefixV1(cid.DagProtobuf, mh.SHA2_256) + dbp := &ihelper.DagBuilderParams{ + Dagserv: n.DAG, + RawLeaves: true, + Maxlinks: ihelper.DefaultLinksPerBlock, + NoCopy: true, + Prefix: &prefix, + URL: url, + } + + blc, err := balanced.Layout(dbp.New(chk)) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + res.SetOutput(BlockStat{ + Key: blc.Cid().String(), + Size: int(hres.ContentLength), + }) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + bs := res.Output().(*BlockStat) + return strings.NewReader(bs.Key + "\n"), nil + }, + }, +} diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 88cd9e4a764..5cfbbb48640 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -19,12 +19,12 @@ import ( "github.com/ipfs/go-ipfs/pin" unixfs "github.com/ipfs/go-ipfs/unixfs" - posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" chunker "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files" + posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" bstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ) diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 791289e2f39..1f2debc2159 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -18,9 +18,9 @@ import ( "github.com/ipfs/go-ipfs/repo/config" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" - pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files" + pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" datastore "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" syncds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync" diff --git a/filestore/filestore.go b/filestore/filestore.go index 66289b0686f..6140451ed21 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -11,9 +11,9 @@ import ( "context" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" - posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" + posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" dsq "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/query" ) diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index 9589c3dd5fa..6323f49c874 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -9,8 +9,8 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" - posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" + posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" ) diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index 2e481ade90d..6d4509876f2 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "net/http" "os" "path/filepath" @@ -12,8 +13,8 @@ import ( dshelp "gx/ipfs/QmNP2u7bofwUQptHQGPfabGWtTCbxhNLSZKqbf1uzsup9V/go-ipfs-ds-help" proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" - posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" + posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" dsns "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/namespace" @@ -111,7 +112,6 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) { if err != nil { return nil, err } - out, err := f.readDataObj(c, dobj) if err != nil { return nil, err @@ -120,6 +120,14 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) { return blocks.NewBlockWithCid(out, c) } +func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { + if !d.GetURL() { + return f.readFileDataObj(c, d) + } else { + return f.readURLDataObj(c, d) + } +} + func (f *FileManager) getDataObj(c *cid.Cid) (*pb.DataObj, error) { o, err := f.ds.Get(dshelp.CidToDsKey(c)) switch err { @@ -148,8 +156,7 @@ func unmarshalDataObj(o interface{}) (*pb.DataObj, error) { return &dobj, nil } -// reads and verifies the block -func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { +func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { p := filepath.FromSlash(d.GetFilePath()) abspath := filepath.Join(f.root, p) @@ -187,6 +194,46 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { return outbuf, nil } +// reads and verifies the block from URL +func (f *FileManager) readURLDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { + + req, err := http.NewRequest("GET", d.GetFilePath(), nil) + if err != nil { + return nil, err + } + + req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", d.GetOffset(), d.GetOffset()+d.GetSize_()-1)) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusPartialContent { + return nil, fmt.Errorf("expected HTTP 206 got %d", res.StatusCode) + } + + outbuf := make([]byte, d.GetSize_()) + _, err = io.ReadFull(res.Body, outbuf) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil, &CorruptReferenceError{StatusFileChanged, err} + } else if err != nil { + return nil, &CorruptReferenceError{StatusFileError, err} + } + res.Body.Close() + + outcid, err := c.Prefix().Sum(outbuf) + if err != nil { + return nil, err + } + + if !c.Equals(outcid) { + return nil, &CorruptReferenceError{StatusFileChanged, + fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} + } + + return outbuf, nil +} + // Has returns if the FileManager is storing a block reference. It does not // validate the data, nor checks if the reference is valid. func (f *FileManager) Has(c *cid.Cid) (bool, error) { @@ -209,16 +256,21 @@ func (f *FileManager) Put(b *posinfo.FilestoreNode) error { func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { var dobj pb.DataObj - if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) { - return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root) - } + if !b.PosInfo.IsURL { + if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) { + return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root) + } - p, err := filepath.Rel(f.root, b.PosInfo.FullPath) - if err != nil { - return err - } + p, err := filepath.Rel(f.root, b.PosInfo.FullPath) + if err != nil { + return err + } - dobj.FilePath = proto.String(filepath.ToSlash(p)) + dobj.FilePath = proto.String(filepath.ToSlash(p)) + } else { + dobj.FilePath = proto.String(b.PosInfo.FullPath) + dobj.URL = proto.Bool(true) + } dobj.Offset = proto.Uint64(b.PosInfo.Offset) dobj.Size_ = proto.Uint64(uint64(len(b.RawData()))) diff --git a/filestore/pb/Makefile b/filestore/pb/Makefile index 5101a482d20..505f70e7541 100644 --- a/filestore/pb/Makefile +++ b/filestore/pb/Makefile @@ -1,10 +1,8 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) +include mk/header.mk -all: $(GO) +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) -%.pb.go: %.proto - protoc --gogo_out=. $< +#DEPS_GO += $(TGTS_$(d)) -clean: - rm *.pb.go +include mk/footer.mk diff --git a/filestore/pb/dataobj.pb.go b/filestore/pb/dataobj.pb.go index fadd40c1a48..517ae1b7f5c 100644 --- a/filestore/pb/dataobj.pb.go +++ b/filestore/pb/dataobj.pb.go @@ -1,12 +1,12 @@ // Code generated by protoc-gen-gogo. -// source: dataobj.proto +// source: filestore/pb/dataobj.proto // DO NOT EDIT! /* Package datastore_pb is a generated protocol buffer package. It is generated from these files: - dataobj.proto + filestore/pb/dataobj.proto It has these top-level messages: DataObj @@ -26,6 +26,7 @@ type DataObj struct { FilePath *string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath,omitempty"` Offset *uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset,omitempty"` Size_ *uint64 `protobuf:"varint,3,opt,name=Size" json:"Size,omitempty"` + URL *bool `protobuf:"varint,4,opt,name=URL" json:"URL,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -54,6 +55,13 @@ func (m *DataObj) GetSize_() uint64 { return 0 } +func (m *DataObj) GetURL() bool { + if m != nil && m.URL != nil { + return *m.URL + } + return false +} + func init() { proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj") } diff --git a/filestore/pb/dataobj.proto b/filestore/pb/dataobj.proto index c7d7f0eea8f..311042a0ed8 100644 --- a/filestore/pb/dataobj.proto +++ b/filestore/pb/dataobj.proto @@ -4,4 +4,5 @@ message DataObj { optional string FilePath = 1; optional uint64 Offset = 2; optional uint64 Size = 3; + optional bool URL = 4; } diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 3e5480fb61a..d58129ab4de 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -25,6 +25,7 @@ type DagBuilderHelper struct { maxlinks int batch *ipld.Batch fullPath string + isUrl bool stat os.FileInfo prefix *cid.Prefix } @@ -48,6 +49,8 @@ type DagBuilderParams struct { // NoCopy signals to the chunker that it should track fileinfo for // filestore adds NoCopy bool + + URL string } // New generates a new DagBuilderHelper from the given params and a given @@ -65,6 +68,11 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper { db.fullPath = fi.AbsPath() db.stat = fi.Stat() } + + if dbp.URL != "" { + db.fullPath = dbp.URL + db.isUrl = true + } return db } @@ -206,7 +214,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { // from the DagBuilderHelper. func (db *DagBuilderHelper) SetPosInfo(node *UnixfsNode, offset uint64) { if db.fullPath != "" { - node.SetPosInfo(offset, db.fullPath, db.stat) + node.SetPosInfo(offset, db.fullPath, db.stat, db.isUrl) } } diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 64a22f5510a..9fae2c7458a 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -8,9 +8,9 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" - pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" + pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" ) // BlockSizeLimit specifies the maximum size an imported block can have. @@ -142,11 +142,12 @@ func (n *UnixfsNode) FileSize() uint64 { // SetPosInfo sets information about the offset of the data of this node in a // filesystem file. -func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo) { +func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo, isUrl bool) { n.posInfo = &pi.PosInfo{ Offset: offset, FullPath: fullPath, Stat: stat, + IsURL: isUrl, } } diff --git a/package.json b/package.json index 748f889b28c..0395f5daeaf 100644 --- a/package.json +++ b/package.json @@ -422,9 +422,9 @@ }, { "author": "hector", - "hash": "QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe", + "hash": "QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC", "name": "go-ipfs-posinfo", - "version": "0.0.2" + "version": "0.0.3" }, { "author": "hsanjuan", From d59a6e99a66166d47bf0fd7b2c7335c9c5357522 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 22 Jun 2018 20:58:02 -0400 Subject: [PATCH 02/13] Fix "ipfs urlstore add" output. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/urlstore.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index 03647e7a7c2..6be89c5533f 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -77,7 +77,9 @@ var urlAdd = &cmds.Command{ }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - bs := res.Output().(*BlockStat) + ch := res.Output().(<-chan interface{}) + bs0 := <-ch + bs := bs0.(*BlockStat) return strings.NewReader(bs.Key + "\n"), nil }, }, From 696a0f039e464f4c8b8db6dfa78c4480d5e268fd Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 22 Jun 2018 21:39:15 -0400 Subject: [PATCH 03/13] Simplify code: use prefix instead of flag to determine if a url License: MIT Signed-off-by: Kevin Atkinson --- core/coreunix/add.go | 2 +- core/coreunix/add_test.go | 2 +- filestore/filestore.go | 2 +- filestore/filestore_test.go | 14 +++++++++++++- filestore/fsrefstore.go | 13 +++++++++---- filestore/pb/Makefile | 12 +++++++----- filestore/pb/dataobj.pb.go | 12 ++---------- filestore/pb/dataobj.proto | 1 - importer/helpers/dagbuilder.go | 4 +--- importer/helpers/helpers.go | 5 ++--- package.json | 4 ++-- 11 files changed, 39 insertions(+), 32 deletions(-) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 5cfbbb48640..88cd9e4a764 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -19,12 +19,12 @@ import ( "github.com/ipfs/go-ipfs/pin" unixfs "github.com/ipfs/go-ipfs/unixfs" + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" chunker "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files" - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" bstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ) diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 1f2debc2159..791289e2f39 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -18,9 +18,9 @@ import ( "github.com/ipfs/go-ipfs/repo/config" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" + pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files" - pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" datastore "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" syncds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync" diff --git a/filestore/filestore.go b/filestore/filestore.go index 6140451ed21..66289b0686f 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -11,9 +11,9 @@ import ( "context" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" dsq "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/query" ) diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index 6323f49c874..8336010e55b 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -9,8 +9,8 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" ) @@ -162,3 +162,15 @@ func TestDeletes(t *testing.T) { } } } + +func TestIsURL(t *testing.T) { + if !IsURL("http://www.example.com") { + t.Fatal("IsURL failed: http://www.example.com") + } + if !IsURL("https://www.example.com") { + t.Fatal("IsURL failed: https://www.example.com") + } + if IsURL("adir/afile") { + t.Fatal("IsURL recognized non-url") + } +} diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index 6d4509876f2..d51139536cc 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -13,8 +13,8 @@ import ( dshelp "gx/ipfs/QmNP2u7bofwUQptHQGPfabGWtTCbxhNLSZKqbf1uzsup9V/go-ipfs-ds-help" proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" dsns "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/namespace" @@ -121,7 +121,7 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) { } func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { - if !d.GetURL() { + if !IsURL(d.GetFilePath()) { return f.readFileDataObj(c, d) } else { return f.readURLDataObj(c, d) @@ -256,7 +256,7 @@ func (f *FileManager) Put(b *posinfo.FilestoreNode) error { func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { var dobj pb.DataObj - if !b.PosInfo.IsURL { + if !IsURL(b.PosInfo.FullPath) { if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) { return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root) } @@ -269,7 +269,6 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { dobj.FilePath = proto.String(filepath.ToSlash(p)) } else { dobj.FilePath = proto.String(b.PosInfo.FullPath) - dobj.URL = proto.Bool(true) } dobj.Offset = proto.Uint64(b.PosInfo.Offset) dobj.Size_ = proto.Uint64(uint64(len(b.RawData()))) @@ -298,3 +297,9 @@ func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error { return batch.Commit() } + +func IsURL(str string) bool { + return (len(str) > 7 && str[0] == 'h' && str[1] == 't' && str[2] == 't' && str[3] == 'p') && + ((len(str) > 8 && str[4] == 's' && str[5] == ':' && str[6] == '/' && str[7] == '/') || + (str[4] == ':' && str[5] == '/' && str[6] == '/')) +} diff --git a/filestore/pb/Makefile b/filestore/pb/Makefile index 505f70e7541..5101a482d20 100644 --- a/filestore/pb/Makefile +++ b/filestore/pb/Makefile @@ -1,8 +1,10 @@ -include mk/header.mk +PB = $(wildcard *.proto) +GO = $(PB:.proto=.pb.go) -PB_$(d) = $(wildcard $(d)/*.proto) -TGTS_$(d) = $(PB_$(d):.proto=.pb.go) +all: $(GO) -#DEPS_GO += $(TGTS_$(d)) +%.pb.go: %.proto + protoc --gogo_out=. $< -include mk/footer.mk +clean: + rm *.pb.go diff --git a/filestore/pb/dataobj.pb.go b/filestore/pb/dataobj.pb.go index 517ae1b7f5c..fadd40c1a48 100644 --- a/filestore/pb/dataobj.pb.go +++ b/filestore/pb/dataobj.pb.go @@ -1,12 +1,12 @@ // Code generated by protoc-gen-gogo. -// source: filestore/pb/dataobj.proto +// source: dataobj.proto // DO NOT EDIT! /* Package datastore_pb is a generated protocol buffer package. It is generated from these files: - filestore/pb/dataobj.proto + dataobj.proto It has these top-level messages: DataObj @@ -26,7 +26,6 @@ type DataObj struct { FilePath *string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath,omitempty"` Offset *uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset,omitempty"` Size_ *uint64 `protobuf:"varint,3,opt,name=Size" json:"Size,omitempty"` - URL *bool `protobuf:"varint,4,opt,name=URL" json:"URL,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -55,13 +54,6 @@ func (m *DataObj) GetSize_() uint64 { return 0 } -func (m *DataObj) GetURL() bool { - if m != nil && m.URL != nil { - return *m.URL - } - return false -} - func init() { proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj") } diff --git a/filestore/pb/dataobj.proto b/filestore/pb/dataobj.proto index 311042a0ed8..c7d7f0eea8f 100644 --- a/filestore/pb/dataobj.proto +++ b/filestore/pb/dataobj.proto @@ -4,5 +4,4 @@ message DataObj { optional string FilePath = 1; optional uint64 Offset = 2; optional uint64 Size = 3; - optional bool URL = 4; } diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index d58129ab4de..5dac1b56961 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -25,7 +25,6 @@ type DagBuilderHelper struct { maxlinks int batch *ipld.Batch fullPath string - isUrl bool stat os.FileInfo prefix *cid.Prefix } @@ -71,7 +70,6 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper { if dbp.URL != "" { db.fullPath = dbp.URL - db.isUrl = true } return db } @@ -214,7 +212,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { // from the DagBuilderHelper. func (db *DagBuilderHelper) SetPosInfo(node *UnixfsNode, offset uint64) { if db.fullPath != "" { - node.SetPosInfo(offset, db.fullPath, db.stat, db.isUrl) + node.SetPosInfo(offset, db.fullPath, db.stat) } } diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 9fae2c7458a..64a22f5510a 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -8,9 +8,9 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" + pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" - pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" ) // BlockSizeLimit specifies the maximum size an imported block can have. @@ -142,12 +142,11 @@ func (n *UnixfsNode) FileSize() uint64 { // SetPosInfo sets information about the offset of the data of this node in a // filesystem file. -func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo, isUrl bool) { +func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo) { n.posInfo = &pi.PosInfo{ Offset: offset, FullPath: fullPath, Stat: stat, - IsURL: isUrl, } } diff --git a/package.json b/package.json index 0395f5daeaf..748f889b28c 100644 --- a/package.json +++ b/package.json @@ -422,9 +422,9 @@ }, { "author": "hector", - "hash": "QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC", + "hash": "QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe", "name": "go-ipfs-posinfo", - "version": "0.0.3" + "version": "0.0.2" }, { "author": "hsanjuan", From b53a1b3022bdb5a0b480ecb4fb87826db73fdc7e Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sat, 23 Jun 2018 17:03:57 -0400 Subject: [PATCH 04/13] Add config option to enable urlstore. License: MIT Signed-off-by: Kevin Atkinson --- core/builder.go | 2 +- docs/experimental-features.md | 21 +++++++++++++++++++++ filestore/filestore_test.go | 1 + filestore/fsrefstore.go | 21 ++++++++++++++++++--- repo/config/experiments.go | 1 + repo/fsrepo/fsrepo.go | 4 +++- 6 files changed, 45 insertions(+), 5 deletions(-) diff --git a/core/builder.go b/core/builder.go index 2cd3bb6ea6f..f0e0bb81464 100644 --- a/core/builder.go +++ b/core/builder.go @@ -213,7 +213,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { n.GCLocker = bstore.NewGCLocker() n.Blockstore = bstore.NewGCBlockstore(cbs, n.GCLocker) - if conf.Experimental.FilestoreEnabled { + if conf.Experimental.FilestoreEnabled || conf.Experimental.UrlstoreEnabled { // hash security n.Filestore = filestore.NewFilestore(cbs, n.Repo.FileManager()) n.Blockstore = bstore.NewGCBlockstore(n.Filestore, n.GCLocker) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 7e916e950ba..aeb3fd23b48 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -17,6 +17,7 @@ the above issue. - [go-multiplex stream muxer](#go-multiplex-stream-muxer) - [Raw leaves for unixfs files](#raw-leaves-for-unixfs-files) - [ipfs filestore](#ipfs-filestore) +- [ipfs urlstore](#ipfs-urlstore) - [BadgerDB datastore](#badger-datastore) - [Private Networks](#private-networks) - [ipfs p2p](#ipfs-p2p) @@ -164,6 +165,26 @@ And then pass the `--nocopy` flag when running `ipfs add` --- +## ipfs urlstore +Allows ipfs to retrieve blocks contents via a url instead of storing it in the datastore + +### State +experimental. + +### In Version +???. + +### How to enable +Modify your ipfs config: +``` +ipfs config --json Experimental.UrlstoreEnabled true +``` + +### Road to being a real feature +???. + +--- + ## Private Networks Allows ipfs to only connect to other peers who have a shared secret key. diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index 8336010e55b..7f15e90097f 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -23,6 +23,7 @@ func newTestFilestore(t *testing.T) (string, *Filestore) { t.Fatal(err) } fm := NewFileManager(mds, testdir) + fm.AllowFiles = true bs := blockstore.NewBlockstore(mds) fstore := NewFilestore(bs, fm) diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index d51139536cc..02770eee759 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -29,8 +29,10 @@ var FilestorePrefix = ds.NewKey("filestore") // to the actual location of the block data in the filesystem // (a path and an offset). type FileManager struct { - ds ds.Batching - root string + AllowFiles bool + AllowUrls bool + ds ds.Batching + root string } // CorruptReferenceError implements the error interface. @@ -52,7 +54,7 @@ func (c CorruptReferenceError) Error() string { // datastore and root. All FilestoreNodes paths are relative to the // root path given here, which is prepended for any operations. func NewFileManager(ds ds.Batching, root string) *FileManager { - return &FileManager{dsns.Wrap(ds, FilestorePrefix), root} + return &FileManager{ds: dsns.Wrap(ds, FilestorePrefix), root: root} } // AllKeysChan returns a channel from which to read the keys stored in @@ -157,6 +159,10 @@ func unmarshalDataObj(o interface{}) (*pb.DataObj, error) { } func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { + if !f.AllowFiles { + return nil, fmt.Errorf("filestore not enabled") + } + p := filepath.FromSlash(d.GetFilePath()) abspath := filepath.Join(f.root, p) @@ -196,6 +202,9 @@ func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) // reads and verifies the block from URL func (f *FileManager) readURLDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { + if !f.AllowUrls { + return nil, fmt.Errorf("urlstore not enabled") + } req, err := http.NewRequest("GET", d.GetFilePath(), nil) if err != nil { @@ -257,6 +266,9 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { var dobj pb.DataObj if !IsURL(b.PosInfo.FullPath) { + if !f.AllowFiles { + return fmt.Errorf("filestore not enabled") + } if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) { return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root) } @@ -268,6 +280,9 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { dobj.FilePath = proto.String(filepath.ToSlash(p)) } else { + if !f.AllowUrls { + return fmt.Errorf("urlstore not enabled") + } dobj.FilePath = proto.String(b.PosInfo.FullPath) } dobj.Offset = proto.Uint64(b.PosInfo.Offset) diff --git a/repo/config/experiments.go b/repo/config/experiments.go index f76572ee2af..ab48c868159 100644 --- a/repo/config/experiments.go +++ b/repo/config/experiments.go @@ -2,6 +2,7 @@ package config type Experiments struct { FilestoreEnabled bool + UrlstoreEnabled bool ShardingEnabled bool Libp2pStreamMounting bool } diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 7d53e226397..64bd53ce9e9 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -175,8 +175,10 @@ func open(repoPath string) (repo.Repo, error) { return nil, err } - if r.config.Experimental.FilestoreEnabled { + if r.config.Experimental.FilestoreEnabled || r.config.Experimental.UrlstoreEnabled { r.filemgr = filestore.NewFileManager(r.ds, filepath.Dir(r.path)) + r.filemgr.AllowFiles = r.config.Experimental.FilestoreEnabled + r.filemgr.AllowUrls = r.config.Experimental.UrlstoreEnabled } keepLocked = true From 90972095a9ca0ea2f6ab6580b28bc5ac635596a2 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 26 Jun 2018 00:35:46 -0400 Subject: [PATCH 05/13] Add test cases for urlstore. License: MIT Signed-off-by: Kevin Atkinson --- test/sharness/t0272-urlstore.sh | 69 +++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 test/sharness/t0272-urlstore.sh diff --git a/test/sharness/t0272-urlstore.sh b/test/sharness/t0272-urlstore.sh new file mode 100755 index 00000000000..da2eeec3dc1 --- /dev/null +++ b/test/sharness/t0272-urlstore.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2017 Jeromy Johnson +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test out the urlstore functionality" + +. lib/test-lib.sh + +test_init_ipfs + +test_expect_success "enable urlstore" ' + ipfs config --json Experimental.UrlstoreEnabled true +' + +test_expect_success "create some random files" ' + random 2222 7 > file1 && + random 50000000 7 > file2 +' + +test_expect_success "add files using trickle dag format without raw leaves" ' + HASH1a=$(ipfs add -q --trickle --raw-leaves=false file1) && + HASH2a=$(ipfs add -q --trickle --raw-leaves=false file2) +' +test_launch_ipfs_daemon --offline + +test_expect_success "make sure files can be retrived via the gateway" ' + curl http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a -o file1.actual && + test_cmp file1 file1.actual && + curl http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a -o file2.actual && + test_cmp file2 file2.actual +' + +test_expect_success "add files using gateway address via url store" ' + HASH1=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) && + HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) +' + +test_expect_success "make sure hashes are different" ' + echo $HASH1a $HASH1 + echo $HASH2a $HASH2 +' + +test_expect_success "get files via urlstore" ' + ipfs get $HASH1 -o file1.actual && + test_cmp file1 file1.actual && + ipfs get $HASH2 -o file2.actual && + test_cmp file2 file2.actual +' + +test_expect_success "remove original hashes from local gateway" ' + ipfs pin rm $HASH1a $HASH2a && + ipfs repo gc +' + +test_expect_success "gatway no longer has files" ' + test_must_fail curl -f http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a -o file1.actual + test_must_fail curl -f http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a -o file2.actual +' + +test_expect_success "files can not be retrieved via the urlstore" ' + test_must_fail ipfs get $HASH1 + test_must_fail ipfs get $HASH2 +' + +test_kill_ipfs_daemon + +test_done From e5189f423013cb03949f33e232aa6f7dd43298d3 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 26 Jun 2018 04:56:03 -0400 Subject: [PATCH 06/13] Return better error code when an http request failed. License: MIT Signed-off-by: Kevin Atkinson --- filestore/fsrefstore.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index 02770eee759..710deac0321 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -215,10 +215,11 @@ func (f *FileManager) readURLDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) res, err := http.DefaultClient.Do(req) if err != nil { - return nil, err + return nil, &CorruptReferenceError{StatusFileError, err} } if res.StatusCode != http.StatusPartialContent { - return nil, fmt.Errorf("expected HTTP 206 got %d", res.StatusCode) + return nil, &CorruptReferenceError{StatusFileError, + fmt.Errorf("expected HTTP 206 got %d", res.StatusCode)} } outbuf := make([]byte, d.GetSize_()) From b3457f240cc1d4d9e0e9399303ae7cfbd6905727 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 26 Jun 2018 15:44:57 -0400 Subject: [PATCH 07/13] Enhance tests. License: MIT Signed-off-by: Kevin Atkinson --- test/sharness/t0272-urlstore.sh | 71 ++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/test/sharness/t0272-urlstore.sh b/test/sharness/t0272-urlstore.sh index da2eeec3dc1..e143edba43a 100755 --- a/test/sharness/t0272-urlstore.sh +++ b/test/sharness/t0272-urlstore.sh @@ -16,12 +16,14 @@ test_expect_success "enable urlstore" ' test_expect_success "create some random files" ' random 2222 7 > file1 && - random 50000000 7 > file2 + random 500000 7 > file2 && + random 50000000 7 > file3 ' test_expect_success "add files using trickle dag format without raw leaves" ' HASH1a=$(ipfs add -q --trickle --raw-leaves=false file1) && - HASH2a=$(ipfs add -q --trickle --raw-leaves=false file2) + HASH2a=$(ipfs add -q --trickle --raw-leaves=false file2) && + HASH3a=$(ipfs add -q --trickle --raw-leaves=false file3) ' test_launch_ipfs_daemon --offline @@ -29,17 +31,20 @@ test_expect_success "make sure files can be retrived via the gateway" ' curl http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a -o file1.actual && test_cmp file1 file1.actual && curl http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a -o file2.actual && - test_cmp file2 file2.actual + test_cmp file2 file2.actual && + curl http://127.0.0.1:$GWAY_PORT/ipfs/$HASH3a -o file3.actual && + test_cmp file3 file3.actual ' test_expect_success "add files using gateway address via url store" ' HASH1=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) && - HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) + HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) && + ipfs pin add $HASH1 $HASH2 ' test_expect_success "make sure hashes are different" ' - echo $HASH1a $HASH1 - echo $HASH2a $HASH2 + echo $HASH1a $HASH1 ## FIXME + echo $HASH2a $HASH2 ## FIXME ' test_expect_success "get files via urlstore" ' @@ -49,9 +54,31 @@ test_expect_success "get files via urlstore" ' test_cmp file2 file2.actual ' +cat < ls_expect +zb2rhX1q5oFFzEkPNsTe1Y8osUdFqSQGjUWRZsqC9fbY6WVSk 262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 0 +zb2rhYbKFn1UWGHXaAitcdVTkDGTykX8RFpGWzRFuLpoe9VE4 237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 262144 +zb2rhjddJ5DNzBrFu8G6CP1ApY25BukwCeskXHzN1H18CiVVZ 2222 http://127.0.0.1:$GWAY_PORT/ipfs/QmcHm3BL2cXuQ6rJdKQgPrmT9suqGkfy2KzH3MkXPEBXU6 0 +EOF + +test_expect_success "ipfs filestore ls works with urls" ' + ipfs filestore ls | sort > ls_actual && + test_cmp ls_expect ls_actual +' + +cat < verify_expect +ok zb2rhX1q5oFFzEkPNsTe1Y8osUdFqSQGjUWRZsqC9fbY6WVSk 262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 0 +ok zb2rhYbKFn1UWGHXaAitcdVTkDGTykX8RFpGWzRFuLpoe9VE4 237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 262144 +ok zb2rhjddJ5DNzBrFu8G6CP1ApY25BukwCeskXHzN1H18CiVVZ 2222 http://127.0.0.1:$GWAY_PORT/ipfs/QmcHm3BL2cXuQ6rJdKQgPrmT9suqGkfy2KzH3MkXPEBXU6 0 +EOF + +test_expect_success "ipfs filestore verify works with urls" ' + ipfs filestore verify | sort > verify_actual && + test_cmp verify_expect verify_actual +' + test_expect_success "remove original hashes from local gateway" ' ipfs pin rm $HASH1a $HASH2a && - ipfs repo gc + ipfs repo gc > /dev/null ' test_expect_success "gatway no longer has files" ' @@ -59,11 +86,37 @@ test_expect_success "gatway no longer has files" ' test_must_fail curl -f http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a -o file2.actual ' -test_expect_success "files can not be retrieved via the urlstore" ' - test_must_fail ipfs get $HASH1 +cat < verify_expect_2 +error zb2rhX1q5oFFzEkPNsTe1Y8osUdFqSQGjUWRZsqC9fbY6WVSk 262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 0 +error zb2rhYbKFn1UWGHXaAitcdVTkDGTykX8RFpGWzRFuLpoe9VE4 237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 262144 +error zb2rhjddJ5DNzBrFu8G6CP1ApY25BukwCeskXHzN1H18CiVVZ 2222 http://127.0.0.1:$GWAY_PORT/ipfs/QmcHm3BL2cXuQ6rJdKQgPrmT9suqGkfy2KzH3MkXPEBXU6 0 +EOF + +test_expect_success "ipfs filestore verify is correct" ' + ipfs filestore verify | sort > verify_actual_2 && + test_cmp verify_expect_2 verify_actual_2 +' + +test_expect_failure "files can not be retrieved via the urlstore" ' + test_must_fail ipfs get $HASH1 && test_must_fail ipfs get $HASH2 ' +test_expect_success "add large file using gateway address via url store" ' + HASH3=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH3a) +' + +test_expect_success "get large file via urlstore" ' + ipfs get $HASH3 -o file3.actual && + test_cmp file3 file3.actual +' + test_kill_ipfs_daemon +test_expect_success "files can not be retrieved via the urlstore" ' + test_must_fail ipfs get $HASH1 && + test_must_fail ipfs get $HASH2 && + test_must_fail ipfs get $HASH3 +' + test_done From 0e2444428042023dfeca787d532dad5bc54a1778 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 26 Jun 2018 15:43:56 -0400 Subject: [PATCH 08/13] Add some documentation to `ipfs urlstore add` command. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/urlstore.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index 6be89c5533f..58f9d6aa5d8 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -24,6 +24,25 @@ var UrlStoreCmd = &cmds.Command{ } var urlAdd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Add URLs via urlstore.", + LongDescription: ` +Add URLs to ipfs without storing the data locally. + +The URL provided must be stable and ideally on a web server under your +control. + +The file is added using raw-leaves but otherwise using the default +settings for 'ipfs add'. + +The file is not pinned, so this command should be followed by an 'ipfs +pin add'. + +This command is considered temporary until a better solution can be +found. It may disappear or the semantics can change at any +time. +`, + }, Arguments: []cmdkit.Argument{ cmdkit.StringArg("url", true, false, "URL to add to IPFS"), }, From ed2bb81b8d0951fe8433d39cb8ff9d433b3a90b2 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 26 Jun 2018 15:55:36 -0400 Subject: [PATCH 09/13] Code cleanups to make code climate happy. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/root.go | 2 +- core/commands/urlstore.go | 2 +- filestore/fsrefstore.go | 19 ++++++++++--------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/core/commands/root.go b/core/commands/root.go index 1f62b720520..648a2a88610 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -136,7 +136,7 @@ var rootSubcommands = map[string]*cmds.Command{ "tar": lgc.NewCommand(TarCmd), "file": lgc.NewCommand(unixfs.UnixFSCmd), "update": lgc.NewCommand(ExternalBinary()), - "urlstore": lgc.NewCommand(UrlStoreCmd), + "urlstore": lgc.NewCommand(urlStoreCmd), "version": lgc.NewCommand(VersionCmd), "shutdown": lgc.NewCommand(daemonShutdownCmd), } diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index 58f9d6aa5d8..8818e33b7d6 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -16,7 +16,7 @@ import ( cmdkit "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" ) -var UrlStoreCmd = &cmds.Command{ +var urlStoreCmd = &cmds.Command{ Subcommands: map[string]*cmds.Command{ "add": urlAdd, diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index 710deac0321..be0be92c7d1 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -123,11 +123,10 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) { } func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { - if !IsURL(d.GetFilePath()) { - return f.readFileDataObj(c, d) - } else { + if IsURL(d.GetFilePath()) { return f.readURLDataObj(c, d) } + return f.readFileDataObj(c, d) } func (f *FileManager) getDataObj(c *cid.Cid) (*pb.DataObj, error) { @@ -266,7 +265,12 @@ func (f *FileManager) Put(b *posinfo.FilestoreNode) error { func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { var dobj pb.DataObj - if !IsURL(b.PosInfo.FullPath) { + if IsURL(b.PosInfo.FullPath) { + if !f.AllowUrls { + return fmt.Errorf("urlstore not enabled") + } + dobj.FilePath = proto.String(b.PosInfo.FullPath) + } else { if !f.AllowFiles { return fmt.Errorf("filestore not enabled") } @@ -280,11 +284,6 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { } dobj.FilePath = proto.String(filepath.ToSlash(p)) - } else { - if !f.AllowUrls { - return fmt.Errorf("urlstore not enabled") - } - dobj.FilePath = proto.String(b.PosInfo.FullPath) } dobj.Offset = proto.Uint64(b.PosInfo.Offset) dobj.Size_ = proto.Uint64(uint64(len(b.RawData()))) @@ -314,6 +313,8 @@ func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error { return batch.Commit() } +// IsURL returns true if the string represents a valid URL that the +// urlstore can handle. func IsURL(str string) bool { return (len(str) > 7 && str[0] == 'h' && str[1] == 't' && str[2] == 't' && str[3] == 'p') && ((len(str) > 8 && str[4] == 's' && str[5] == ':' && str[6] == '/' && str[7] == '/') || From 0c2efb90d3896194ee18f28449503963972209cf Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 26 Jun 2018 19:15:44 -0400 Subject: [PATCH 10/13] More test fixes. License: MIT Signed-off-by: Kevin Atkinson --- test/sharness/t0272-urlstore.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/sharness/t0272-urlstore.sh b/test/sharness/t0272-urlstore.sh index e143edba43a..dbffc142328 100755 --- a/test/sharness/t0272-urlstore.sh +++ b/test/sharness/t0272-urlstore.sh @@ -43,8 +43,8 @@ test_expect_success "add files using gateway address via url store" ' ' test_expect_success "make sure hashes are different" ' - echo $HASH1a $HASH1 ## FIXME - echo $HASH2a $HASH2 ## FIXME + test $HASH1a != $HASH1 && + test $HASH2a != $HASH2 ' test_expect_success "get files via urlstore" ' @@ -97,15 +97,19 @@ test_expect_success "ipfs filestore verify is correct" ' test_cmp verify_expect_2 verify_actual_2 ' -test_expect_failure "files can not be retrieved via the urlstore" ' - test_must_fail ipfs get $HASH1 && - test_must_fail ipfs get $HASH2 +test_expect_success "files can not be retrieved via the urlstore" ' + test_must_fail ipfs cat $HASH1 > /dev/null && + test_must_fail ipfs cat $HASH2 > /dev/null ' test_expect_success "add large file using gateway address via url store" ' HASH3=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH3a) ' +test_expect_success "make sure hashes are different" ' + test $HASH3a != $HASH3 +' + test_expect_success "get large file via urlstore" ' ipfs get $HASH3 -o file3.actual && test_cmp file3 file3.actual @@ -114,9 +118,9 @@ test_expect_success "get large file via urlstore" ' test_kill_ipfs_daemon test_expect_success "files can not be retrieved via the urlstore" ' - test_must_fail ipfs get $HASH1 && - test_must_fail ipfs get $HASH2 && - test_must_fail ipfs get $HASH3 + test_must_fail ipfs cat $HASH1 > /dev/null && + test_must_fail ipfs cat $HASH2 > /dev/null && + test_must_fail ipfs cat $HASH3 > /dev/null ' test_done From 6a4b1262a5fde7025095fe9f26ab117dd27f2e3b Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 26 Jun 2018 21:41:33 -0400 Subject: [PATCH 11/13] Make sure you can't add URL's unless the url store is enabled. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/urlstore.go | 11 +++++++++++ test/sharness/t0272-urlstore.sh | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index 8818e33b7d6..a0a25be73a6 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -56,6 +56,17 @@ time. return } + cfg, err := n.Repo.Config() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + if !cfg.Experimental.UrlstoreEnabled { + res.SetError(fmt.Errorf("URL store not enabled."), cmdkit.ErrNormal) + return + } + hreq, err := http.NewRequest("GET", url, nil) if err != nil { res.SetError(err, cmdkit.ErrNormal) diff --git a/test/sharness/t0272-urlstore.sh b/test/sharness/t0272-urlstore.sh index dbffc142328..312247f2f07 100755 --- a/test/sharness/t0272-urlstore.sh +++ b/test/sharness/t0272-urlstore.sh @@ -10,10 +10,6 @@ test_description="Test out the urlstore functionality" test_init_ipfs -test_expect_success "enable urlstore" ' - ipfs config --json Experimental.UrlstoreEnabled true -' - test_expect_success "create some random files" ' random 2222 7 > file1 && random 500000 7 > file2 && @@ -36,6 +32,19 @@ test_expect_success "make sure files can be retrived via the gateway" ' test_cmp file3 file3.actual ' +test_expect_success "add files without enabling url store" ' + test_must_fail ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a && + test_must_fail ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a +' + +test_kill_ipfs_daemon + +test_expect_success "enable urlstore" ' + ipfs config --json Experimental.UrlstoreEnabled true +' + +test_launch_ipfs_daemon --offline + test_expect_success "add files using gateway address via url store" ' HASH1=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) && HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) && From 8dd970b73aa48af72c85185d3f9edbaac2dd081e Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 26 Jun 2018 22:30:15 -0400 Subject: [PATCH 12/13] filestore: Return consistent err msg. when file/urlstore is not enabled. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/add.go | 4 ++-- core/commands/filestore.go | 2 +- core/commands/urlstore.go | 3 ++- filestore/filestore.go | 4 ++++ filestore/fsrefstore.go | 8 ++++---- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 43520bcca97..46204c06db2 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -10,6 +10,7 @@ import ( blockservice "github.com/ipfs/go-ipfs/blockservice" core "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/coreunix" + filestore "github.com/ipfs/go-ipfs/filestore" dag "github.com/ipfs/go-ipfs/merkledag" dagtest "github.com/ipfs/go-ipfs/merkledag/test" mfs "github.com/ipfs/go-ipfs/mfs" @@ -183,8 +184,7 @@ You can now check what blocks have been created by: // nocopy -> filestoreEnabled if nocopy && !cfg.Experimental.FilestoreEnabled { - res.SetError(errors.New("filestore is not enabled, see https://git.io/vNItf"), - cmdkit.ErrClient) + res.SetError(filestore.ErrFilestoreNotEnabled, cmdkit.ErrClient) return } diff --git a/core/commands/filestore.go b/core/commands/filestore.go index effd28e9e7d..f24be141076 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -237,7 +237,7 @@ func getFilestore(env interface{}) (*core.IpfsNode, *filestore.Filestore, error) } fs := n.Filestore if fs == nil { - return n, nil, fmt.Errorf("filestore not enabled") + return n, nil, filestore.ErrFilestoreNotEnabled } return n, fs, err } diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index a0a25be73a6..80f9eb679cf 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -7,6 +7,7 @@ import ( "strings" cmds "github.com/ipfs/go-ipfs/commands" + filestore "github.com/ipfs/go-ipfs/filestore" balanced "github.com/ipfs/go-ipfs/importer/balanced" ihelper "github.com/ipfs/go-ipfs/importer/helpers" @@ -63,7 +64,7 @@ time. } if !cfg.Experimental.UrlstoreEnabled { - res.SetError(fmt.Errorf("URL store not enabled."), cmdkit.ErrNormal) + res.SetError(filestore.ErrUrlstoreNotEnabled, cmdkit.ErrNormal) return } diff --git a/filestore/filestore.go b/filestore/filestore.go index 66289b0686f..0c80fb1d59a 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -9,6 +9,7 @@ package filestore import ( "context" + "errors" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" @@ -20,6 +21,9 @@ import ( var log = logging.Logger("filestore") +var ErrFilestoreNotEnabled = errors.New("filestore is not enabled, see https://git.io/vNItf") +var ErrUrlstoreNotEnabled = errors.New("urlstore is not enabled") + // Filestore implements a Blockstore by combining a standard Blockstore // to store regular blocks and a special Blockstore called // FileManager to store blocks which data exists in an external file. diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index be0be92c7d1..98255caa78d 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -159,7 +159,7 @@ func unmarshalDataObj(o interface{}) (*pb.DataObj, error) { func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { if !f.AllowFiles { - return nil, fmt.Errorf("filestore not enabled") + return nil, ErrFilestoreNotEnabled } p := filepath.FromSlash(d.GetFilePath()) @@ -202,7 +202,7 @@ func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) // reads and verifies the block from URL func (f *FileManager) readURLDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { if !f.AllowUrls { - return nil, fmt.Errorf("urlstore not enabled") + return nil, ErrUrlstoreNotEnabled } req, err := http.NewRequest("GET", d.GetFilePath(), nil) @@ -267,12 +267,12 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { if IsURL(b.PosInfo.FullPath) { if !f.AllowUrls { - return fmt.Errorf("urlstore not enabled") + return ErrUrlstoreNotEnabled } dobj.FilePath = proto.String(b.PosInfo.FullPath) } else { if !f.AllowFiles { - return fmt.Errorf("filestore not enabled") + return ErrFilestoreNotEnabled } if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) { return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root) From 1f29699d90a595b861ffacd5795ad15133d6dab6 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 29 Jun 2018 23:03:51 -0400 Subject: [PATCH 13/13] Address c.r. and additional tweaks. License: MIT Signed-off-by: Kevin Atkinson --- core/commands/urlstore.go | 7 ++++++- filestore/filestore_test.go | 2 +- filestore/fsrefstore.go | 3 ++- importer/helpers/dagbuilder.go | 5 ++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go index 80f9eb679cf..b903f95ed8f 100644 --- a/core/commands/urlstore.go +++ b/core/commands/urlstore.go @@ -26,7 +26,7 @@ var urlStoreCmd = &cmds.Command{ var urlAdd = &cmds.Command{ Helptext: cmdkit.HelpText{ - Tagline: "Add URLs via urlstore.", + Tagline: "Add URL via urlstore.", LongDescription: ` Add URLs to ipfs without storing the data locally. @@ -57,6 +57,11 @@ time. return } + if !filestore.IsURL(url) { + res.SetError(fmt.Errorf("unsupported url syntax: %s", url), cmdkit.ErrNormal) + return + } + cfg, err := n.Repo.Config() if err != nil { res.SetError(err, cmdkit.ErrNormal) diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index 7f15e90097f..279a2bc8218 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -171,7 +171,7 @@ func TestIsURL(t *testing.T) { if !IsURL("https://www.example.com") { t.Fatal("IsURL failed: https://www.example.com") } - if IsURL("adir/afile") { + if IsURL("adir/afile") || IsURL("http:/ /afile") || IsURL("http:/a/file") { t.Fatal("IsURL recognized non-url") } } diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index 98255caa78d..960fc93e834 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -314,7 +314,8 @@ func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error { } // IsURL returns true if the string represents a valid URL that the -// urlstore can handle. +// urlstore can handle. More specifically it returns true if a string +// begins with 'http://' or 'https://'. func IsURL(str string) bool { return (len(str) > 7 && str[0] == 'h' && str[1] == 't' && str[2] == 't' && str[3] == 'p') && ((len(str) > 8 && str[4] == 's' && str[5] == ':' && str[6] == '/' && str[7] == '/') || diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 5dac1b56961..eaa11d0d66b 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -49,6 +49,9 @@ type DagBuilderParams struct { // filestore adds NoCopy bool + // URL if non-empty (and NoCopy is also true) indicates that the + // file will not be stored in the datastore but instead retrieved + // from this location via the urlstore. URL string } @@ -68,7 +71,7 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper { db.stat = fi.Stat() } - if dbp.URL != "" { + if dbp.URL != "" && dbp.NoCopy { db.fullPath = dbp.URL } return db