From a7407d2d80d5658bb4cb1767df6e37481cb846be Mon Sep 17 00:00:00 2001 From: Nate White Date: Mon, 18 May 2020 11:34:34 -0700 Subject: [PATCH] Reorganize to reduce import size of core modules. This is a huge change intended to be a proof-of-concept of reorganizing key parts of Noms to allow its use by roci.dev/replicache-client to fit within ~1MB of Brotli-compressed WASM. It is being shared to allow for directional review of the scope and types of changes needed to achieve this end, but should be broken up and reviewed as ~6 smaller commits of more limited scope. Major changes: o go/d: Replace use of testify/assert with hand-rolled functions in go/d/assert.go. Replace some previously-used functions with inline checks to keep number of replacements down. Also replaced Equal/NotEqual() with StringEqual/NotEqual() since this covered most cases and avoided the need for a lot of cases in the testify code. Saves 4,328,755 bytes (uncompressed) in repjs js/wasm. o Separate flag registration from go/util/verbose to go/util/verbose/verboseflags to eliminate kingpin dependency for simple users of verbose package. o Move http server and handlers from go/datas to go/datas/remote, adjust all references. Some previously private elements moved to go/datas/internal so they could be used from datas/ and datas/remote but not outside of the datas/ directory. o Move chunks/chunk_store_common_test.go to a new chunks/chunkstest package so it can be used outside of the chunks package. o Move parts of go/chunks/test_utils.go (file name did not restrict this just to tests) that use testify/assert to chunks/chunkstest, and other parts to chunks/memory_store.go. o Break up go/spec so that core pieces are in go/spec/lite (package name spec), but no protocol types are registered. Users may import (as _) various packages under go/spec (e.g., go/spec/nbs, aws, http) to install those protocols. go/spec itself forwards most types and functions from go/spec/lite *and* imports all submodules so that unmodified users of go/spec retain full compatibility. o Add build tags to go/types/graph_builder, opcache, opcache_compare exclude them from the js/wasm build. These files should be moved to a new a new package (e.g., go/types/graph), but there are various private package members shared between these and the rest of go/types that need to be untangled first. Pending changes: o go/d: Move CheckError() to new package go/d/check_error to avoid import of kingpin, or, alternatively, require binaries using CheckError() to register a Usage() callback to avoid the direct kingpin dependency. o Restore functionality of RemoteDatabaseSuite currently in go/datas/database_test.go. There's a thorny knot to unwind to allow DatabaseSuite access to unexported members of go/datas while also being able to use go/datas/remote without creating an import cycle. --- cmd/noms/noms.go | 4 +- cmd/noms/noms_serve.go | 4 +- go.mod | 13 +- go.sum | 41 +- go/chunks/chunk_serializer.go | 9 +- .../chunk_store_common.go} | 23 +- go/chunks/memory_store.go | 40 ++ go/chunks/memory_store_test.go | 9 +- go/chunks/test_utils.go | 85 ---- go/d/assert.go | 264 +++++++++++++ go/d/check_error.go | 4 +- go/d/try.go | 6 +- go/datas/commit.go | 12 +- go/datas/database_common.go | 2 + go/datas/database_test.go | 14 +- go/datas/internal/internal.go | 52 +++ go/datas/{ => internal}/serialize_hashes.go | 8 +- .../{ => internal}/serialize_hashes_test.go | 8 +- go/datas/pull.go | 5 + go/datas/{ => remote}/database_server.go | 2 +- go/datas/{ => remote}/http_chunk_store.go | 11 +- .../{ => remote}/http_chunk_store_test.go | 9 +- go/datas/{ => remote}/pull_test.go | 47 +-- .../{ => remote}/remote_database_handlers.go | 21 +- .../remote_database_handlers_test.go | 18 +- go/perf/suite/suite.go | 5 +- go/spec/aws/aws.go | 55 +++ go/spec/aws/aws_test.go | 26 ++ go/spec/http/http.go | 36 ++ go/spec/http/http_test.go | 27 ++ go/spec/{ => lite}/absolute_path.go | 0 go/spec/{ => lite}/commit_meta.go | 0 go/spec/{ => lite}/commit_meta_test.go | 0 go/spec/lite/lite_test.go | 33 ++ go/spec/lite/spec.go | 324 +++++++++++++++ go/spec/{ => lite}/util.go | 0 go/spec/nbs/nbs.go | 25 ++ go/spec/nbs/nbs_test.go | 27 ++ go/spec/spec.go | 372 ++---------------- go/types/encode_human_readable.go | 4 +- go/types/graph_builder.go | 2 + go/types/opcache.go | 2 + go/types/opcache_compare.go | 2 + go/types/opcache_test.go | 13 +- go/types/subtype_test.go | 7 +- go/util/verbose/verbose.go | 11 - go/util/verbose/verbose_test.go | 42 ++ go/util/verbose/verboseflags/register.go | 24 ++ samples/go/counter/counter.go | 4 +- samples/go/csv/csv-export/exporter.go | 4 +- samples/go/csv/csv-import/importer.go | 4 +- samples/go/hr/main.go | 4 +- samples/go/nomdex/nomdex.go | 4 +- samples/go/nomsfs/nomsfs.go | 24 +- samples/go/xml-import/xml_importer.go | 10 +- 55 files changed, 1226 insertions(+), 576 deletions(-) rename go/chunks/{chunk_store_common_test.go => chunkstest/chunk_store_common.go} (79%) delete mode 100644 go/chunks/test_utils.go create mode 100644 go/d/assert.go create mode 100644 go/datas/internal/internal.go rename go/datas/{ => internal}/serialize_hashes.go (84%) rename go/datas/{ => internal}/serialize_hashes_test.go (86%) rename go/datas/{ => remote}/database_server.go (99%) rename go/datas/{ => remote}/http_chunk_store.go (98%) rename go/datas/{ => remote}/http_chunk_store_test.go (98%) rename go/datas/{ => remote}/pull_test.go (88%) rename go/datas/{ => remote}/remote_database_handlers.go (98%) rename go/datas/{ => remote}/remote_database_handlers_test.go (96%) create mode 100644 go/spec/aws/aws.go create mode 100644 go/spec/aws/aws_test.go create mode 100644 go/spec/http/http.go create mode 100644 go/spec/http/http_test.go rename go/spec/{ => lite}/absolute_path.go (100%) rename go/spec/{ => lite}/commit_meta.go (100%) rename go/spec/{ => lite}/commit_meta_test.go (100%) create mode 100644 go/spec/lite/lite_test.go create mode 100644 go/spec/lite/spec.go rename go/spec/{ => lite}/util.go (100%) create mode 100644 go/spec/nbs/nbs.go create mode 100644 go/spec/nbs/nbs_test.go create mode 100644 go/util/verbose/verbose_test.go create mode 100644 go/util/verbose/verboseflags/register.go diff --git a/cmd/noms/noms.go b/cmd/noms/noms.go index f4edc7f795..2e35279e90 100644 --- a/cmd/noms/noms.go +++ b/cmd/noms/noms.go @@ -16,7 +16,7 @@ import ( "github.com/attic-labs/noms/cmd/noms/splore" "github.com/attic-labs/noms/cmd/util" "github.com/attic-labs/noms/go/util/profile" - "github.com/attic-labs/noms/go/util/verbose" + "github.com/attic-labs/noms/go/util/verbose/verboseflags" ) var kingpinCommands = []util.KingpinCommand{ @@ -67,7 +67,7 @@ func main() { // global flags profile.RegisterProfileFlags(noms) - verbose.RegisterVerboseFlags(noms) + verboseflags.Register(noms) handlers := map[string]util.KingpinHandler{} diff --git a/cmd/noms/noms_serve.go b/cmd/noms/noms_serve.go index 6d55dc2d3e..50f37ff5e9 100644 --- a/cmd/noms/noms_serve.go +++ b/cmd/noms/noms_serve.go @@ -13,7 +13,7 @@ import ( "github.com/attic-labs/noms/cmd/util" "github.com/attic-labs/noms/go/config" "github.com/attic-labs/noms/go/d" - "github.com/attic-labs/noms/go/datas" + "github.com/attic-labs/noms/go/datas/remote" "github.com/attic-labs/noms/go/util/profile" ) @@ -26,7 +26,7 @@ func nomsServe(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandl cfg := config.NewResolver() cs, err := cfg.GetChunkStore(*db) d.CheckError(err) - server := datas.NewRemoteDatabaseServer(cs, *port) + server := remote.NewRemoteDatabaseServer(cs, *port) // Shutdown server gracefully so that profile may be written c := make(chan os.Signal, 1) diff --git a/go.mod b/go.mod index c81a01cbb3..6e4d6d6561 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,7 @@ go 1.12 require ( github.com/BurntSushi/toml v0.3.1 - github.com/aboodman/noms-gx v0.0.0-20180714061401-d6cb97cb040b - github.com/alecthomas/kingpin v2.2.6+incompatible // indirect + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect github.com/attic-labs/graphql v0.0.0-20190507195614-b6552d20145f @@ -14,21 +13,21 @@ require ( github.com/clbanning/mxj v1.8.4 github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf github.com/dustin/go-humanize v1.0.0 + github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/snappy v0.0.1 github.com/hanwen/go-fuse v1.0.0 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 - github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d github.com/julienschmidt/httprouter v1.2.0 github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mattn/go-colorable v0.1.1 // indirect github.com/mattn/go-isatty v0.0.7 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b github.com/shirou/gopsutil v2.18.12+incompatible + github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e github.com/stretchr/testify v1.3.0 github.com/syndtr/goleveldb v1.0.0 - golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c - golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a - golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 - gopkg.in/alecthomas/kingpin.v2 v2.2.6 + golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect + golang.org/x/sys v0.0.0-20190412213103-97732733099d ) diff --git a/go.sum b/go.sum index 727e83ab0c..ffaf71c34e 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,13 @@ -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/aboodman/noms-gx v0.0.0-20180714061401-d6cb97cb040b h1:6pe29GhapzKB4egv5gr8JHHFiExf2m81JJw4BKDBC6g= -github.com/aboodman/noms-gx v0.0.0-20180714061401-d6cb97cb040b/go.mod h1:ni7quUEZfdz5Q36a9VJgeUlTaYfwY3fS3j/v5WIz8zs= -github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= -github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/attic-labs/graphql v0.0.0-20190507195614-b6552d20145f h1:WMEteRGdJItAZfxaCPyL6SEfyh4+bE+LsN50UKz46EA= github.com/attic-labs/graphql v0.0.0-20190507195614-b6552d20145f/go.mod h1:1U3eDKPYQXn3o4jpC2rAlH9THIo+ZOKWSI0FyeG1SEI= -github.com/attic-labs/kingpin v2.2.6+incompatible h1:gzq18qMaCcbpq2ysBO51P6D8bficBtmYk6ZjiSOK/OQ= -github.com/attic-labs/kingpin v2.2.6+incompatible/go.mod h1:Cp18FeDCvsK+cD2QAGkqerGjrgSXLiJWnjHeY2mneBc= github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible h1:wd5mq8xSfwCYd1JpQ309s+3tTlP/gifcG2awOA3x5Vk= github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible/go.mod h1:Cp18FeDCvsK+cD2QAGkqerGjrgSXLiJWnjHeY2mneBc= github.com/aws/aws-sdk-go v1.19.26 h1:GavKlzJDfYQGoS4jn2F+KYYZlR8QEhrLPfpf8+oJhS4= @@ -25,26 +20,31 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= -github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d h1:c93kUJDtVAXFEhsCh5jSxyOJmFHuzcihnslQiX8Urwo= -github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 h1:l6Y3mFnF46A+CeZsTrT8kVIuhayq1266oxWpDKE7hnQ= github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6/go.mod h1:UtDV9qK925GVmbdjR+e1unqoo+wGWNHHC6XB1Eu6wpE= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -54,12 +54,16 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e h1:VAzdS5Nw68fbf5RZ8RDVlUvPXNU6Z3jtPCK/qvm4FoQ= github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -68,26 +72,29 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/go/chunks/chunk_serializer.go b/go/chunks/chunk_serializer.go index 2232b94bb5..cf04800fbe 100644 --- a/go/chunks/chunk_serializer.go +++ b/go/chunks/chunk_serializer.go @@ -13,6 +13,10 @@ import ( "github.com/attic-labs/noms/go/hash" ) +var ( + emptyChunkHash = EmptyChunk.Hash() +) + /* Chunk Serialization: Chunk 0 @@ -55,7 +59,10 @@ func Deserialize(reader io.Reader, chunkChan chan<- *Chunk) (err error) { if err != nil { break } - d.Chk.NotEqual(EmptyChunk.Hash(), c.Hash()) + h := c.Hash() + if bytes.Equal(emptyChunkHash[:], h[:]) { + d.Chk.Fail("Should not be: %#v\n", h) + } chunkChan <- &c } if err == io.EOF { diff --git a/go/chunks/chunk_store_common_test.go b/go/chunks/chunkstest/chunk_store_common.go similarity index 79% rename from go/chunks/chunk_store_common_test.go rename to go/chunks/chunkstest/chunk_store_common.go index 60cb115c6b..2e88fb8ee6 100644 --- a/go/chunks/chunk_store_common_test.go +++ b/go/chunks/chunkstest/chunk_store_common.go @@ -2,24 +2,26 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package chunks +package chunkstest import ( + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/attic-labs/noms/go/chunks" "github.com/attic-labs/noms/go/constants" "github.com/attic-labs/noms/go/hash" ) type ChunkStoreTestSuite struct { suite.Suite - Factory Factory + Factory chunks.Factory } func (suite *ChunkStoreTestSuite) TestChunkStorePut() { store := suite.Factory.CreateStore("ns") input := "abc" - c := NewChunk([]byte(input)) + c := chunks.NewChunk([]byte(input)) store.Put(c) h := c.Hash() @@ -48,7 +50,7 @@ func (suite *ChunkStoreTestSuite) TestChunkStoreCommitPut() { name := "ns" store := suite.Factory.CreateStore(name) input := "abc" - c := NewChunk([]byte(input)) + c := chunks.NewChunk([]byte(input)) store.Put(c) h := c.Hash() @@ -82,7 +84,7 @@ func (suite *ChunkStoreTestSuite) TestChunkStoreVersion() { func (suite *ChunkStoreTestSuite) TestChunkStoreCommitUnchangedRoot() { store1, store2 := suite.Factory.CreateStore("ns"), suite.Factory.CreateStore("ns") input := "abc" - c := NewChunk([]byte(input)) + c := chunks.NewChunk([]byte(input)) store1.Put(c) h := c.Hash() @@ -96,3 +98,14 @@ func (suite *ChunkStoreTestSuite) TestChunkStoreCommitUnchangedRoot() { // Now, reading c from store2 via the API should work... assertInputInStore(input, h, store2, suite.Assert()) } + +func assertInputInStore(input string, h hash.Hash, s chunks.ChunkStore, assert *assert.Assertions) { + chunk := s.Get(h) + assert.False(chunk.IsEmpty(), "Shouldn't get empty chunk for %s", h.String()) + assert.Equal(input, string(chunk.Data())) +} + +func assertInputNotInStore(input string, h hash.Hash, s chunks.ChunkStore, assert *assert.Assertions) { + chunk := s.Get(h) + assert.True(chunk.IsEmpty(), "Shouldn't get non-empty chunk for %s: %v", h.String(), chunk) +} diff --git a/go/chunks/memory_store.go b/go/chunks/memory_store.go index b9525ed6e7..711cbadeaa 100644 --- a/go/chunks/memory_store.go +++ b/go/chunks/memory_store.go @@ -221,3 +221,43 @@ func (f *memoryStoreFactory) CreateStore(ns string) ChunkStore { func (f *memoryStoreFactory) Shutter() { f.stores = nil } + +type TestStorage struct { + MemoryStorage +} + +func (t *TestStorage) NewView() *TestStoreView { + return &TestStoreView{ChunkStore: t.MemoryStorage.NewView()} +} + +type TestStoreView struct { + ChunkStore + Reads int + Hases int + Writes int +} + +func (s *TestStoreView) Get(h hash.Hash) Chunk { + s.Reads++ + return s.ChunkStore.Get(h) +} + +func (s *TestStoreView) GetMany(hashes hash.HashSet, foundChunks chan *Chunk) { + s.Reads += len(hashes) + s.ChunkStore.GetMany(hashes, foundChunks) +} + +func (s *TestStoreView) Has(h hash.Hash) bool { + s.Hases++ + return s.ChunkStore.Has(h) +} + +func (s *TestStoreView) HasMany(hashes hash.HashSet) hash.HashSet { + s.Hases += len(hashes) + return s.ChunkStore.HasMany(hashes) +} + +func (s *TestStoreView) Put(c Chunk) { + s.Writes++ + s.ChunkStore.Put(c) +} diff --git a/go/chunks/memory_store_test.go b/go/chunks/memory_store_test.go index 29c73ccec3..87d7ba5b6e 100644 --- a/go/chunks/memory_store_test.go +++ b/go/chunks/memory_store_test.go @@ -2,12 +2,15 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package chunks +package chunks_test import ( "testing" "github.com/stretchr/testify/suite" + + "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/chunks/chunkstest" ) func TestMemoryStoreTestSuite(t *testing.T) { @@ -15,11 +18,11 @@ func TestMemoryStoreTestSuite(t *testing.T) { } type MemoryStoreTestSuite struct { - ChunkStoreTestSuite + chunkstest.ChunkStoreTestSuite } func (suite *MemoryStoreTestSuite) SetupTest() { - suite.Factory = NewMemoryStoreFactory() + suite.Factory = chunks.NewMemoryStoreFactory() } func (suite *MemoryStoreTestSuite) TearDownTest() { diff --git a/go/chunks/test_utils.go b/go/chunks/test_utils.go deleted file mode 100644 index 63fea6e301..0000000000 --- a/go/chunks/test_utils.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2016 Attic Labs, Inc. All rights reserved. -// Licensed under the Apache License, version 2.0: -// http://www.apache.org/licenses/LICENSE-2.0 - -package chunks - -import ( - "github.com/attic-labs/noms/go/d" - "github.com/attic-labs/noms/go/hash" - "github.com/stretchr/testify/assert" -) - -func assertInputInStore(input string, h hash.Hash, s ChunkStore, assert *assert.Assertions) { - chunk := s.Get(h) - assert.False(chunk.IsEmpty(), "Shouldn't get empty chunk for %s", h.String()) - assert.Equal(input, string(chunk.Data())) -} - -func assertInputNotInStore(input string, h hash.Hash, s ChunkStore, assert *assert.Assertions) { - chunk := s.Get(h) - assert.True(chunk.IsEmpty(), "Shouldn't get non-empty chunk for %s: %v", h.String(), chunk) -} - -type TestStorage struct { - MemoryStorage -} - -func (t *TestStorage) NewView() *TestStoreView { - return &TestStoreView{ChunkStore: t.MemoryStorage.NewView()} -} - -type TestStoreView struct { - ChunkStore - Reads int - Hases int - Writes int -} - -func (s *TestStoreView) Get(h hash.Hash) Chunk { - s.Reads++ - return s.ChunkStore.Get(h) -} - -func (s *TestStoreView) GetMany(hashes hash.HashSet, foundChunks chan *Chunk) { - s.Reads += len(hashes) - s.ChunkStore.GetMany(hashes, foundChunks) -} - -func (s *TestStoreView) Has(h hash.Hash) bool { - s.Hases++ - return s.ChunkStore.Has(h) -} - -func (s *TestStoreView) HasMany(hashes hash.HashSet) hash.HashSet { - s.Hases += len(hashes) - return s.ChunkStore.HasMany(hashes) -} - -func (s *TestStoreView) Put(c Chunk) { - s.Writes++ - s.ChunkStore.Put(c) -} - -type TestStoreFactory struct { - stores map[string]*TestStorage -} - -func NewTestStoreFactory() *TestStoreFactory { - return &TestStoreFactory{map[string]*TestStorage{}} -} - -func (f *TestStoreFactory) CreateStore(ns string) ChunkStore { - if f.stores == nil { - d.Panic("Cannot use TestStoreFactory after Shutter().") - } - if ts, present := f.stores[ns]; present { - return ts.NewView() - } - f.stores[ns] = &TestStorage{} - return f.stores[ns].NewView() -} - -func (f *TestStoreFactory) Shutter() { - f.stores = nil -} diff --git a/go/d/assert.go b/go/d/assert.go new file mode 100644 index 0000000000..284dcdbbe0 --- /dev/null +++ b/go/d/assert.go @@ -0,0 +1,264 @@ +package d + +// assert.go is a simplified fork of the parts of +// github.com/stretchr/testify/assert used by Noms via calls on d.Chk. The +// subset present here saves approximately 4MB in compiled binaries for users +// of Noms over a direct import of the base package. + +// Testify's LICENSE file is as follows: +// MIT License + +// Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import ( + "bufio" + "bytes" + "fmt" + "runtime" + "strings" + "unicode" + "unicode/utf8" +) + +type testingT interface { + Errorf(format string, args ...interface{}) +} + +type Assertions struct { + t testingT +} + +func (a *Assertions) Fail(failureMessage string, msgAndArgs ...interface{}) bool { + content := []labeledContent{ + {"Error Trace", strings.Join(CallerInfo(), "\n\t\t\t")}, + {"Error", failureMessage}, + } + + // Add test name if the Go version supports it + if n, ok := a.t.(interface { + Name() string + }); ok { + content = append(content, labeledContent{"Test", n.Name()}) + } + + message := messageFromMsgAndArgs(msgAndArgs...) + if len(message) > 0 { + content = append(content, labeledContent{"Messages", message}) + } + + a.t.Errorf("\n%s", ""+labeledOutput(content...)) + return false +} + +// Stolen from the `go test` tool. +// isTest tells whether name looks like a test (or benchmark, according to prefix). +// It is a Test (say) if there is a character after Test that is not a lower-case letter. +// We don't want TesticularCancer. +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { // "Test" is ok + return true + } + rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(rune) +} + +func messageFromMsgAndArgs(msgAndArgs ...interface{}) string { + if len(msgAndArgs) == 0 || msgAndArgs == nil { + return "" + } + if len(msgAndArgs) == 1 { + msg := msgAndArgs[0] + if msgAsStr, ok := msg.(string); ok { + return msgAsStr + } + return fmt.Sprintf("%+v", msg) + } + if len(msgAndArgs) > 1 { + return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + } + return "" +} + +// Aligns the provided message so that all lines after the first line start at the same location as the first line. +// Assumes that the first line starts at the correct location (after carriage return, tab, label, spacer and tab). +// The longestLabelLen parameter specifies the length of the longest label in the output (required becaues this is the +// basis on which the alignment occurs). +func indentMessageLines(message string, longestLabelLen int) string { + outBuf := new(bytes.Buffer) + + for i, scanner := 0, bufio.NewScanner(strings.NewReader(message)); scanner.Scan(); i++ { + // no need to align first line because it starts at the correct location (after the label) + if i != 0 { + // append alignLen+1 spaces to align with "{{longestLabel}}:" before adding tab + outBuf.WriteString("\n\t" + strings.Repeat(" ", longestLabelLen+1) + "\t") + } + outBuf.WriteString(scanner.Text()) + } + + return outBuf.String() +} + +type labeledContent struct { + label string + content string +} + +// labeledOutput returns a string consisting of the provided labeledContent. Each labeled output is appended in the following manner: +// +// \t{{label}}:{{align_spaces}}\t{{content}}\n +// +// The initial carriage return is required to undo/erase any padding added by testing.T.Errorf. The "\t{{label}}:" is for the label. +// If a label is shorter than the longest label provided, padding spaces are added to make all the labels match in length. Once this +// alignment is achieved, "\t{{content}}\n" is added for the output. +// +// If the content of the labeledOutput contains line breaks, the subsequent lines are aligned so that they start at the same location as the first line. +func labeledOutput(content ...labeledContent) string { + longestLabel := 0 + for _, v := range content { + if len(v.label) > longestLabel { + longestLabel = len(v.label) + } + } + var output string + for _, v := range content { + output += "\t" + v.label + ":" + strings.Repeat(" ", longestLabel-len(v.label)) + "\t" + indentMessageLines(v.content, longestLabel) + "\n" + } + return output +} + +/* CallerInfo is necessary because the assert functions use the testing object +internally, causing it to print the file:line of the assert method, rather than where +the problem actually occurred in calling code.*/ + +// CallerInfo returns an array of strings containing the file and line number +// of each stack frame leading from the current test to the assert call that +// failed. +func CallerInfo() []string { + + var pc uintptr + var ok bool + var file string + var line int + var name string + + callers := []string{} + for i := 0; ; i++ { + pc, file, line, ok = runtime.Caller(i) + if !ok { + // The breaks below failed to terminate the loop, and we ran off the + // end of the call stack. + break + } + + // This is a huge edge case, but it will panic if this is the case, see #180 + if file == "" { + break + } + + f := runtime.FuncForPC(pc) + if f == nil { + break + } + name = f.Name() + + // testing.tRunner is the standard library function that calls + // tests. Subtests are called directly by tRunner, without going through + // the Test/Benchmark/Example function that contains the t.Run calls, so + // with subtests we should break when we hit tRunner, without adding it + // to the list of callers. + if name == "testing.tRunner" { + break + } + + parts := strings.Split(file, "/") + file = parts[len(parts)-1] + if len(parts) > 1 { + dir := parts[len(parts)-2] + if (dir != "assert" && dir != "mock" && dir != "require") || file == "mock_test.go" { + callers = append(callers, fmt.Sprintf("%s:%d", file, line)) + } + } + + // Drop the package + segments := strings.Split(name, ".") + name = segments[len(segments)-1] + if isTest(name, "Test") || + isTest(name, "Benchmark") || + isTest(name, "Example") { + break + } + } + + return callers +} + +func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) bool { + if err != nil { + return a.Fail(fmt.Sprintf("Received unexpected error:\n%+v", err), msgAndArgs...) + } + return true +} + +func (a *Assertions) True(value bool, msgAndArgs ...interface{}) bool { + if !value { + return a.Fail("Should be true", msgAndArgs...) + } + return true +} + +func (a *Assertions) False(value bool, msgAndArgs ...interface{}) bool { + if value { + return a.Fail("Should be false", msgAndArgs...) + } + return true +} + +func (a *Assertions) StringEqual(expected, actual string, msgAndArgs ...interface{}) bool { + if expected != actual { + return a.Fail(fmt.Sprintf("Not equal: \n"+ + "expected: %s\n"+ + "actual : %s", expected, actual), msgAndArgs...) + } + return true +} + +func (a *Assertions) StringNotEqual(expected, actual string, msgAndArgs ...interface{}) bool { + if expected == actual { + return a.Fail(fmt.Sprintf("Should not be: %#v\n", actual), msgAndArgs...) + } + return true +} + +func (a *Assertions) NotNil(object interface{}, msgAndArgs ...interface{}) bool { + if object == nil { + return true + } + return a.Fail("Expected value not to be nil.", msgAndArgs...) +} + +func newAssert(t testingT) *Assertions { + return &Assertions{ + t: t, + } +} diff --git a/go/d/check_error.go b/go/d/check_error.go index 146dbaac8a..f9fd226365 100644 --- a/go/d/check_error.go +++ b/go/d/check_error.go @@ -8,14 +8,14 @@ import ( "fmt" "os" - "github.com/attic-labs/kingpin" + //"github.com/attic-labs/kingpin" "github.com/attic-labs/noms/go/util/exit" ) func CheckError(err error) { if err != nil { fmt.Fprintf(os.Stderr, "error: %s\n", err) - kingpin.Usage() + //kingpin.Usage() exit.Fail() } } diff --git a/go/d/try.go b/go/d/try.go index ff455ef1ad..ff06a4a877 100644 --- a/go/d/try.go +++ b/go/d/try.go @@ -9,13 +9,11 @@ import ( "errors" "fmt" "reflect" - - "github.com/stretchr/testify/assert" ) // d.Chk.() -- used in test cases and as assertions var ( - Chk = assert.New(&panicker{}) + Chk = newAssert(&panicker{}) ) type panicker struct { @@ -95,7 +93,7 @@ func Wrap(err error) WrappedError { } st := stackTracer{} - assert := assert.New(&st) + assert := newAssert(&st) assert.Fail(err.Error()) return wrappedError{st.stackTrace, err} diff --git a/go/datas/commit.go b/go/datas/commit.go index 6e93146638..0f2132693f 100644 --- a/go/datas/commit.go +++ b/go/datas/commit.go @@ -8,8 +8,8 @@ import ( "sort" "github.com/attic-labs/noms/go/d" + "github.com/attic-labs/noms/go/datas/internal" "github.com/attic-labs/noms/go/hash" - "github.com/attic-labs/noms/go/nomdl" "github.com/attic-labs/noms/go/types" ) @@ -22,12 +22,6 @@ const ( var commitTemplate = types.MakeStructTemplate(commitName, []string{MetaField, ParentsField, ValueField}) -var valueCommitType = nomdl.MustParseType(`Struct Commit { - meta: Struct {}, - parents: Set>>, - value: Value, -}`) - // NewCommit creates a new commit object. // // A commit has the following type: @@ -126,11 +120,11 @@ func getRefElementType(t *types.Type) *types.Type { } func IsCommitType(t *types.Type) bool { - return types.IsSubtype(valueCommitType, t) + return internal.IsCommitType(t) } func IsCommit(v types.Value) bool { - return types.IsValueSubtypeOf(v, valueCommitType) + return internal.IsCommit(v) } func IsRefOfCommitType(t *types.Type) bool { diff --git a/go/datas/database_common.go b/go/datas/database_common.go index 84d72c350d..a23254de73 100644 --- a/go/datas/database_common.go +++ b/go/datas/database_common.go @@ -35,9 +35,11 @@ type rootTracker interface { func newDatabase(cs chunks.ChunkStore) *database { vs := types.NewValueStore(cs) + /* if _, ok := cs.(*httpChunkStore); ok { vs.SetEnforceCompleteness(false) } + */ return &database{ ValueStore: vs, // ValueStore is responsible for closing |cs| diff --git a/go/datas/database_test.go b/go/datas/database_test.go index aa5cb41663..e59ba22944 100644 --- a/go/datas/database_test.go +++ b/go/datas/database_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/datas/internal" "github.com/attic-labs/noms/go/hash" "github.com/attic-labs/noms/go/merge" "github.com/attic-labs/noms/go/types" @@ -19,9 +20,11 @@ func TestLocalDatabase(t *testing.T) { suite.Run(t, &LocalDatabaseSuite{}) } +/* func TestRemoteDatabase(t *testing.T) { suite.Run(t, &RemoteDatabaseSuite{}) } +*/ func TestValidateRef(t *testing.T) { st := &chunks.TestStorage{} @@ -51,6 +54,8 @@ func (suite *LocalDatabaseSuite) SetupTest() { suite.db = suite.makeDb(suite.storage.NewView()) } +/* +// TODO(nate): Restore RemoteDatabaseSuite. type RemoteDatabaseSuite struct { DatabaseSuite } @@ -62,16 +67,19 @@ func (suite *RemoteDatabaseSuite) SetupTest() { } suite.db = suite.makeDb(suite.storage.NewView()) } +*/ func (suite *DatabaseSuite) TearDownTest() { suite.db.Close() } +/* func (suite *RemoteDatabaseSuite) TestWriteRefToNonexistentValue() { ds := suite.db.GetDataset("foo") r := types.NewRef(types.Bool(true)) suite.Panics(func() { suite.db.CommitValue(ds, r) }) } +*/ func (suite *DatabaseSuite) TestTolerateUngettableRefs() { suite.Nil(suite.db.ReadValue(hash.Hash{})) @@ -226,17 +234,17 @@ func (suite *DatabaseSuite) TestDatasetsMapType() { datasets := suite.db.Datasets() ds, err := suite.db.CommitValue(suite.db.GetDataset(dsID1), types.String("a")) suite.NoError(err) - suite.NotPanics(func() { assertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) }) + suite.NotPanics(func() { internal.AssertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) }) datasets = suite.db.Datasets() _, err = suite.db.CommitValue(suite.db.GetDataset(dsID2), types.Number(42)) suite.NoError(err) - suite.NotPanics(func() { assertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) }) + suite.NotPanics(func() { internal.AssertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) }) datasets = suite.db.Datasets() _, err = suite.db.Delete(ds) suite.NoError(err) - suite.NotPanics(func() { assertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) }) + suite.NotPanics(func() { internal.AssertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) }) } func newOpts(vrw types.ValueReadWriter, parents ...types.Value) CommitOptions { diff --git a/go/datas/internal/internal.go b/go/datas/internal/internal.go new file mode 100644 index 0000000000..5f2ff20e66 --- /dev/null +++ b/go/datas/internal/internal.go @@ -0,0 +1,52 @@ +package internal + +import ( + "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/d" + "github.com/attic-labs/noms/go/nomdl" + "github.com/attic-labs/noms/go/types" +) + +var valueCommitType = nomdl.MustParseType(`Struct Commit { + meta: Struct {}, + parents: Set>>, + value: Value, +}`) + +func IsCommitType(t *types.Type) bool { + return types.IsSubtype(valueCommitType, t) +} + +func IsCommit(v types.Value) bool { + return types.IsValueSubtypeOf(v, valueCommitType) +} + +func PersistChunks(cs chunks.ChunkStore) { + for !cs.Commit(cs.Root(), cs.Root()) { + } +} + +func AssertMapOfStringToRefOfCommit(proposed, datasets types.Map, vr types.ValueReader) { + stopChan := make(chan struct{}) + defer close(stopChan) + changes := make(chan types.ValueChanged) + go func() { + defer close(changes) + proposed.Diff(datasets, changes, stopChan) + }() + for change := range changes { + switch change.ChangeType { + case types.DiffChangeAdded, types.DiffChangeModified: + // Since this is a Map Diff, change.V is the key at which a change was detected. + // Go get the Value there, which should be a Ref, deref it, and then ensure the target is a Commit. + val := change.NewValue + ref, ok := val.(types.Ref) + if !ok { + d.Panic("Root of a Database must be a Map>, but key %s maps to a %s", change.Key.(types.String), types.TypeOf(val).Describe()) + } + if targetValue := ref.TargetValue(vr); !IsCommit(targetValue) { + d.Panic("Root of a Database must be a Map>, but the ref at key %s points to a %s", change.Key.(types.String), types.TypeOf(targetValue).Describe()) + } + } + } +} diff --git a/go/datas/serialize_hashes.go b/go/datas/internal/serialize_hashes.go similarity index 84% rename from go/datas/serialize_hashes.go rename to go/datas/internal/serialize_hashes.go index f9d2cec652..c5c5ccd64d 100644 --- a/go/datas/serialize_hashes.go +++ b/go/datas/internal/serialize_hashes.go @@ -2,7 +2,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package datas +package internal import ( "encoding/binary" @@ -13,11 +13,11 @@ import ( "github.com/attic-labs/noms/go/hash" ) -func serializedLength(batch chunks.ReadBatch) uint32 { +func SerializedLength(batch chunks.ReadBatch) uint32 { return uint32(len(batch)*hash.ByteLen + binary.Size(uint32(0))) } -func serializeHashes(w io.Writer, batch chunks.ReadBatch) { +func SerializeHashes(w io.Writer, batch chunks.ReadBatch) { err := binary.Write(w, binary.BigEndian, uint32(len(batch))) // 4 billion hashes is probably absurd. Maybe this should be smaller? d.PanicIfError(err) for h := range batch { @@ -30,7 +30,7 @@ func serializeHash(w io.Writer, h hash.Hash) { d.PanicIfError(err) } -func deserializeHashes(reader io.Reader) hash.HashSlice { +func DeserializeHashes(reader io.Reader) hash.HashSlice { count := uint32(0) err := binary.Read(reader, binary.BigEndian, &count) d.PanicIfError(err) diff --git a/go/datas/serialize_hashes_test.go b/go/datas/internal/serialize_hashes_test.go similarity index 86% rename from go/datas/serialize_hashes_test.go rename to go/datas/internal/serialize_hashes_test.go index efdb05d814..f93ec68f6d 100644 --- a/go/datas/serialize_hashes_test.go +++ b/go/datas/internal/serialize_hashes_test.go @@ -2,7 +2,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package datas +package internal import ( "bytes" @@ -23,14 +23,14 @@ func TestHashRoundTrip(t *testing.T) { } defer input.Close() - serializeHashes(b, input) + SerializeHashes(b, input) serializedLen := b.Len() - output := deserializeHashes(b) + output := DeserializeHashes(b) assert.Len(t, output, len(input), "Output has different number of elements than input: %v, %v", output, input) for _, h := range output { _, present := input[h] assert.True(t, present, "%s is in output but not in input", h) } - assert.Equal(t, uint32(serializedLen), serializedLength(input)) + assert.Equal(t, uint32(serializedLen), SerializedLength(input)) } diff --git a/go/datas/pull.go b/go/datas/pull.go index 1954ff0ebb..2cbf9a35ee 100644 --- a/go/datas/pull.go +++ b/go/datas/pull.go @@ -10,6 +10,7 @@ import ( "github.com/attic-labs/noms/go/chunks" "github.com/attic-labs/noms/go/d" + "github.com/attic-labs/noms/go/datas/internal" "github.com/attic-labs/noms/go/hash" "github.com/attic-labs/noms/go/types" "github.com/golang/snappy" @@ -100,3 +101,7 @@ func Pull(srcDB, sinkDB Database, sourceRef types.Ref, progressCh chan PullProgr persistChunks(sinkDB.chunkStore()) } + +func persistChunks(cs chunks.ChunkStore) { + internal.PersistChunks(cs) +} diff --git a/go/datas/database_server.go b/go/datas/remote/database_server.go similarity index 99% rename from go/datas/database_server.go rename to go/datas/remote/database_server.go index de7d2973b5..e6775c2280 100644 --- a/go/datas/database_server.go +++ b/go/datas/remote/database_server.go @@ -2,7 +2,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package datas +package remote import ( "fmt" diff --git a/go/datas/http_chunk_store.go b/go/datas/remote/http_chunk_store.go similarity index 98% rename from go/datas/http_chunk_store.go rename to go/datas/remote/http_chunk_store.go index 84ccdbbf29..09c5d218fb 100644 --- a/go/datas/http_chunk_store.go +++ b/go/datas/remote/http_chunk_store.go @@ -2,7 +2,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package datas +package remote import ( "bufio" @@ -20,6 +20,7 @@ import ( "github.com/attic-labs/noms/go/chunks" "github.com/attic-labs/noms/go/constants" "github.com/attic-labs/noms/go/d" + "github.com/attic-labs/noms/go/datas/internal" "github.com/attic-labs/noms/go/hash" "github.com/attic-labs/noms/go/nbs" "github.com/attic-labs/noms/go/util/verbose" @@ -309,7 +310,7 @@ func (hcs *httpChunkStore) getRefs(batch chunks.ReadBatch) { "Accept-Encoding": {"x-snappy-framed"}, "Content-Type": {"application/octet-stream"}, }) - req.ContentLength = int64(serializedLength(batch)) + req.ContentLength = int64(internal.SerializedLength(batch)) res, err := hcs.httpClient.Do(req) d.Chk.NoError(err) @@ -340,7 +341,7 @@ func (hcs *httpChunkStore) hasRefs(batch chunks.ReadBatch) { "Accept-Encoding": {"x-snappy-framed"}, "Content-Type": {"application/octet-stream"}, }) - req.ContentLength = int64(serializedLength(batch)) + req.ContentLength = int64(internal.SerializedLength(batch)) res, err := hcs.httpClient.Do(req) d.Chk.NoError(err) @@ -550,3 +551,7 @@ func closeResponse(rc io.ReadCloser) error { // d.PanicIfFalse(0 == len(data), string(data)) return rc.Close() } + +func persistChunks(cs chunks.ChunkStore) { + internal.PersistChunks(cs) +} diff --git a/go/datas/http_chunk_store_test.go b/go/datas/remote/http_chunk_store_test.go similarity index 98% rename from go/datas/http_chunk_store_test.go rename to go/datas/remote/http_chunk_store_test.go index c03a3fa0a8..dda257dc94 100644 --- a/go/datas/http_chunk_store_test.go +++ b/go/datas/remote/http_chunk_store_test.go @@ -2,7 +2,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package datas +package remote import ( "encoding/binary" @@ -14,6 +14,7 @@ import ( "github.com/attic-labs/noms/go/chunks" "github.com/attic-labs/noms/go/constants" + "github.com/attic-labs/noms/go/datas" "github.com/attic-labs/noms/go/hash" "github.com/attic-labs/noms/go/types" "github.com/julienschmidt/httprouter" @@ -190,7 +191,7 @@ func (suite *HTTPChunkStoreSuite) TestStats() { func (suite *HTTPChunkStoreSuite) TestRebase() { suite.Equal(hash.Hash{}, suite.http.Root()) - db := NewDatabase(suite.serverCS) + db := datas.NewDatabase(suite.serverCS) defer db.Close() c := types.EncodeValue(types.NewMap(db)) suite.serverCS.Put(c) @@ -202,7 +203,7 @@ func (suite *HTTPChunkStoreSuite) TestRebase() { } func (suite *HTTPChunkStoreSuite) TestRoot() { - db := NewDatabase(suite.serverCS) + db := datas.NewDatabase(suite.serverCS) defer db.Close() c := types.EncodeValue(types.NewMap(db)) suite.serverCS.Put(c) @@ -220,7 +221,7 @@ func (suite *HTTPChunkStoreSuite) TestVersionMismatch() { } func (suite *HTTPChunkStoreSuite) TestCommit() { - db := NewDatabase(suite.serverCS) + db := datas.NewDatabase(suite.serverCS) defer db.Close() c := types.EncodeValue(types.NewMap(db)) suite.serverCS.Put(c) diff --git a/go/datas/pull_test.go b/go/datas/remote/pull_test.go similarity index 88% rename from go/datas/pull_test.go rename to go/datas/remote/pull_test.go index 42f539f15f..ee663ab920 100644 --- a/go/datas/pull_test.go +++ b/go/datas/remote/pull_test.go @@ -2,12 +2,13 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package datas +package remote import ( "testing" "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/datas" "github.com/attic-labs/noms/go/types" "github.com/stretchr/testify/suite" ) @@ -34,8 +35,8 @@ type PullSuite struct { suite.Suite sinkCS *chunks.TestStoreView sourceCS *chunks.TestStoreView - sink Database - source Database + sink datas.Database + source datas.Database commitReads int // The number of reads triggered by commit differs across chunk store impls } @@ -50,8 +51,8 @@ type LocalToLocalSuite struct { func (suite *LocalToLocalSuite) SetupTest() { suite.sinkCS, suite.sourceCS = makeTestStoreViews() - suite.sink = NewDatabase(suite.sinkCS) - suite.source = NewDatabase(suite.sourceCS) + suite.sink = datas.NewDatabase(suite.sinkCS) + suite.source = datas.NewDatabase(suite.sourceCS) } type RemoteToLocalSuite struct { @@ -60,7 +61,7 @@ type RemoteToLocalSuite struct { func (suite *RemoteToLocalSuite) SetupTest() { suite.sinkCS, suite.sourceCS = makeTestStoreViews() - suite.sink = NewDatabase(suite.sinkCS) + suite.sink = datas.NewDatabase(suite.sinkCS) suite.source = makeRemoteDb(suite.sourceCS) } @@ -71,7 +72,7 @@ type LocalToRemoteSuite struct { func (suite *LocalToRemoteSuite) SetupTest() { suite.sinkCS, suite.sourceCS = makeTestStoreViews() suite.sink = makeRemoteDb(suite.sinkCS) - suite.source = NewDatabase(suite.sourceCS) + suite.source = datas.NewDatabase(suite.sourceCS) suite.commitReads = 1 } @@ -86,8 +87,8 @@ func (suite *RemoteToRemoteSuite) SetupTest() { suite.commitReads = 1 } -func makeRemoteDb(cs chunks.ChunkStore) Database { - return NewDatabase(newHTTPChunkStoreForTest(cs)) +func makeRemoteDb(cs chunks.ChunkStore) datas.Database { + return datas.NewDatabase(newHTTPChunkStoreForTest(cs)) } func (suite *PullSuite) TearDownTest() { @@ -98,14 +99,14 @@ func (suite *PullSuite) TearDownTest() { } type progressTracker struct { - Ch chan PullProgress - doneCh chan []PullProgress + Ch chan datas.PullProgress + doneCh chan []datas.PullProgress } func startProgressTracker() *progressTracker { - pt := &progressTracker{make(chan PullProgress), make(chan []PullProgress)} + pt := &progressTracker{make(chan datas.PullProgress), make(chan []datas.PullProgress)} go func() { - progress := []PullProgress{} + progress := []datas.PullProgress{} for info := range pt.Ch { progress = append(progress, info) } @@ -152,13 +153,13 @@ func (suite *PullSuite) TestPullEverything() { sourceRef := suite.commitToSource(l, types.NewSet(suite.source)) pt := startProgressTracker() - Pull(suite.source, suite.sink, sourceRef, pt.Ch) + datas.Pull(suite.source, suite.sink, sourceRef, pt.Ch) suite.True(expectedReads-suite.sinkCS.Reads <= suite.commitReads) pt.Validate(suite) v := suite.sink.ReadValue(sourceRef.TargetHash()).(types.Struct) suite.NotNil(v) - suite.True(l.Equals(v.Get(ValueField))) + suite.True(l.Equals(v.Get(datas.ValueField))) } // Source: -6-> C3(L5) -1-> N @@ -194,14 +195,14 @@ func (suite *PullSuite) TestPullMultiGeneration() { pt := startProgressTracker() - Pull(suite.source, suite.sink, sourceRef, pt.Ch) + datas.Pull(suite.source, suite.sink, sourceRef, pt.Ch) suite.True(expectedReads-suite.sinkCS.Reads <= suite.commitReads) pt.Validate(suite) v := suite.sink.ReadValue(sourceRef.TargetHash()).(types.Struct) suite.NotNil(v) - suite.True(srcL.Equals(v.Get(ValueField))) + suite.True(srcL.Equals(v.Get(datas.ValueField))) } // Source: -6-> C2(L5) -1-> N @@ -240,14 +241,14 @@ func (suite *PullSuite) TestPullDivergentHistory() { pt := startProgressTracker() - Pull(suite.source, suite.sink, sourceRef, pt.Ch) + datas.Pull(suite.source, suite.sink, sourceRef, pt.Ch) suite.True(preReads-suite.sinkCS.Reads <= suite.commitReads) pt.Validate(suite) v := suite.sink.ReadValue(sourceRef.TargetHash()).(types.Struct) suite.NotNil(v) - suite.True(srcL.Equals(v.Get(ValueField))) + suite.True(srcL.Equals(v.Get(datas.ValueField))) } // Source: -6-> C2(L4) -1-> N @@ -282,26 +283,26 @@ func (suite *PullSuite) TestPullUpdates() { pt := startProgressTracker() - Pull(suite.source, suite.sink, sourceRef, pt.Ch) + datas.Pull(suite.source, suite.sink, sourceRef, pt.Ch) suite.True(expectedReads-suite.sinkCS.Reads <= suite.commitReads) pt.Validate(suite) v := suite.sink.ReadValue(sourceRef.TargetHash()).(types.Struct) suite.NotNil(v) - suite.True(srcL.Equals(v.Get(ValueField))) + suite.True(srcL.Equals(v.Get(datas.ValueField))) } func (suite *PullSuite) commitToSource(v types.Value, p types.Set) types.Ref { ds := suite.source.GetDataset(datasetID) - ds, err := suite.source.Commit(ds, v, CommitOptions{Parents: p}) + ds, err := suite.source.Commit(ds, v, datas.CommitOptions{Parents: p}) suite.NoError(err) return ds.HeadRef() } func (suite *PullSuite) commitToSink(v types.Value, p types.Set) types.Ref { ds := suite.sink.GetDataset(datasetID) - ds, err := suite.sink.Commit(ds, v, CommitOptions{Parents: p}) + ds, err := suite.sink.Commit(ds, v, datas.CommitOptions{Parents: p}) suite.NoError(err) return ds.HeadRef() } diff --git a/go/datas/remote_database_handlers.go b/go/datas/remote/remote_database_handlers.go similarity index 98% rename from go/datas/remote_database_handlers.go rename to go/datas/remote/remote_database_handlers.go index 8c20243e8b..6be1b8ac2b 100644 --- a/go/datas/remote_database_handlers.go +++ b/go/datas/remote/remote_database_handlers.go @@ -2,7 +2,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package datas +package remote import ( "compress/gzip" @@ -21,6 +21,8 @@ import ( "github.com/attic-labs/noms/go/chunks" "github.com/attic-labs/noms/go/constants" "github.com/attic-labs/noms/go/d" + "github.com/attic-labs/noms/go/datas" + "github.com/attic-labs/noms/go/datas/internal" "github.com/attic-labs/noms/go/hash" "github.com/attic-labs/noms/go/ngql" "github.com/attic-labs/noms/go/types" @@ -252,11 +254,6 @@ func (wc wc) Close() error { return nil } -func persistChunks(cs chunks.ChunkStore) { - for !cs.Commit(cs.Root(), cs.Root()) { - } -} - func handleGetRefs(w http.ResponseWriter, req *http.Request, ps URLParams, cs chunks.ChunkStore) { if req.Method != "POST" { d.Panic("Expected post method.") @@ -327,7 +324,7 @@ func extractHashes(req *http.Request) hash.HashSlice { reader := bodyReader(req) defer reader.Close() defer io.Copy(ioutil.Discard, reader) // Ensure all data on reader is consumed - return deserializeHashes(reader) + return internal.DeserializeHashes(reader) } func BuildHashesRequestForTest(hashes hash.HashSet) io.ReadCloser { @@ -342,7 +339,7 @@ func buildHashesRequest(batch chunks.ReadBatch) io.ReadCloser { body, pw := io.Pipe() go func() { defer checkClose(pw) - serializeHashes(pw, batch) + internal.SerializeHashes(pw, batch) }() return body } @@ -405,7 +402,7 @@ func handleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, cs c proposedMap := validateProposed(proposed, last, vs) if !proposedMap.Empty() { - assertMapOfStringToRefOfCommit(proposedMap, lastMap, vs) + internal.AssertMapOfStringToRefOfCommit(proposedMap, lastMap, vs) } // If some other client has committed to |vs| since it had |from| at the @@ -475,6 +472,7 @@ func validateProposed(proposed, last hash.Hash, vrw types.ValueReadWriter) types return proposedMap } +/* func assertMapOfStringToRefOfCommit(proposed, datasets types.Map, vr types.ValueReader) { stopChan := make(chan struct{}) defer close(stopChan) @@ -493,12 +491,13 @@ func assertMapOfStringToRefOfCommit(proposed, datasets types.Map, vr types.Value if !ok { d.Panic("Root of a Database must be a Map>, but key %s maps to a %s", change.Key.(types.String), types.TypeOf(val).Describe()) } - if targetValue := ref.TargetValue(vr); !IsCommit(targetValue) { + if targetValue := ref.TargetValue(vr); !datas.IsCommit(targetValue) { d.Panic("Root of a Database must be a Map>, but the ref at key %s points to a %s", change.Key.(types.String), types.TypeOf(targetValue).Describe()) } } } } +*/ func mergeDatasetMaps(a, b, parent types.Map, vrw types.ValueReadWriter) (types.Map, error) { aChangeChan, bChangeChan := make(chan types.ValueChanged), make(chan types.ValueChanged) @@ -603,7 +602,7 @@ func handleGraphQL(w http.ResponseWriter, req *http.Request, ps URLParams, cs ch } // Note: we don't close this becaues |cs| will be closed by the generic endpoint handler - db := NewDatabase(cs) + db := datas.NewDatabase(cs) var rootValue types.Value var err error diff --git a/go/datas/remote_database_handlers_test.go b/go/datas/remote/remote_database_handlers_test.go similarity index 96% rename from go/datas/remote_database_handlers_test.go rename to go/datas/remote/remote_database_handlers_test.go index 23e91946a8..e3cad70d15 100644 --- a/go/datas/remote_database_handlers_test.go +++ b/go/datas/remote/remote_database_handlers_test.go @@ -2,7 +2,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -package datas +package remote import ( "bufio" @@ -17,6 +17,8 @@ import ( "testing" "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/datas" + "github.com/attic-labs/noms/go/datas/internal" "github.com/attic-labs/noms/go/hash" "github.com/attic-labs/noms/go/types" "github.com/golang/snappy" @@ -26,7 +28,7 @@ import ( func TestHandleWriteValue(t *testing.T) { assert := assert.New(t) storage := &chunks.TestStorage{} - db := NewDatabase(storage.NewView()) + db := datas.NewDatabase(storage.NewView()) l := types.NewList( db, @@ -50,7 +52,7 @@ func TestHandleWriteValue(t *testing.T) { HandleWriteValue(w, newRequest("POST", "", "", body, nil), params{}, storage.NewView()) if assert.Equal(http.StatusCreated, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) { - db2 := NewDatabase(storage.NewView()) + db2 := datas.NewDatabase(storage.NewView()) v := db2.ReadValue(l2.Hash()) if assert.NotNil(v) { assert.True(v.Equals(l2), "%+v != %+v", v, l2) @@ -74,7 +76,7 @@ func TestHandleWriteValuePanic(t *testing.T) { func TestHandleWriteValueDupChunks(t *testing.T) { assert := assert.New(t) storage := &chunks.MemoryStorage{} - db := NewDatabase(storage.NewView()) + db := datas.NewDatabase(storage.NewView()) defer db.Close() newItem := types.NewEmptyBlob(db) @@ -90,7 +92,7 @@ func TestHandleWriteValueDupChunks(t *testing.T) { HandleWriteValue(w, newRequest("POST", "", "", body, nil), params{}, storage.NewView()) if assert.Equal(http.StatusCreated, w.Code, "Handler error:\n%s", string(w.Body.Bytes())) { - db := NewDatabase(storage.NewView()) + db := datas.NewDatabase(storage.NewView()) v := db.ReadValue(newItem.Hash()) if assert.NotNil(v) { assert.True(v.Equals(newItem), "%+v != %+v", v, newItem) @@ -143,7 +145,7 @@ func TestBuildHashesRequest(t *testing.T) { } r := buildHashesRequest(batch) defer r.Close() - requested := deserializeHashes(r) + requested := internal.DeserializeHashes(r) for _, h := range requested { _, present := batch[h] @@ -200,7 +202,7 @@ func TestHandleGetBlob(t *testing.T) { blobContents := "I am a blob" storage := &chunks.MemoryStorage{} - db := NewDatabase(storage.NewView()) + db := datas.NewDatabase(storage.NewView()) ds := db.GetDataset("foo") // Test missing h @@ -383,7 +385,7 @@ func buildPostRootURL(current, last hash.Hash) string { } func buildTestCommit(vrw types.ValueReadWriter, v types.Value, parents ...types.Value) types.Struct { - return NewCommit(v, types.NewSet(vrw, parents...), types.NewStruct("Meta", types.StructData{})) + return datas.NewCommit(v, types.NewSet(vrw, parents...), types.NewStruct("Meta", types.StructData{})) } func TestRejectPostRoot(t *testing.T) { diff --git a/go/perf/suite/suite.go b/go/perf/suite/suite.go index a213c2fdae..bc023c307f 100644 --- a/go/perf/suite/suite.go +++ b/go/perf/suite/suite.go @@ -87,6 +87,7 @@ import ( "github.com/attic-labs/noms/go/chunks" "github.com/attic-labs/noms/go/d" "github.com/attic-labs/noms/go/datas" + "github.com/attic-labs/noms/go/datas/remote" "github.com/attic-labs/noms/go/marshal" "github.com/attic-labs/noms/go/nbs" "github.com/attic-labs/noms/go/spec" @@ -273,7 +274,7 @@ func Run(datasetID string, t *testing.T, suiteT perfSuiteT) { serverHost, stopServerFn := suite.StartRemoteDatabase() suite.DatabaseSpec = serverHost - suite.Database = datas.NewDatabase(datas.NewHTTPChunkStore(serverHost, "")) + suite.Database = datas.NewDatabase(remote.NewHTTPChunkStore(serverHost, "")) defer suite.Database.Close() if t, ok := suiteT.(SetupRepSuite); ok { @@ -499,7 +500,7 @@ func (suite *PerfSuite) StartRemoteDatabase() (host string, stopFn func()) { chunkStore = nbs.NewLocalStore(dbDir, 128*(1<<20)) } - server := datas.NewRemoteDatabaseServer(chunkStore, 0) + server := remote.NewRemoteDatabaseServer(chunkStore, 0) portChan := make(chan int) server.Ready = func() { portChan <- server.Port() } go server.Run() diff --git a/go/spec/aws/aws.go b/go/spec/aws/aws.go new file mode 100644 index 0000000000..21d98a613c --- /dev/null +++ b/go/spec/aws/aws.go @@ -0,0 +1,55 @@ +// Copyright 2016 Attic Labs, Inc. All rights reserved. +// Licensed under the Apache License, version 2.0: +// http://www.apache.org/licenses/LICENSE-2.0 + +// Package spec provides builders and parsers for spelling Noms databases, +// datasets and values. +package aws + +import ( + "errors" + "regexp" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/d" + "github.com/attic-labs/noms/go/datas" + "github.com/attic-labs/noms/go/nbs" + "github.com/attic-labs/noms/go/spec/lite" +) + +var GetAWSSession func() *session.Session = func() *session.Session { + return session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2"))) +} + +type awsProtocol struct{} + +var pattern = regexp.MustCompile("^[^/]+/[^/]+/.*$") + +func (t *awsProtocol) Parse(name string) (string, error) { + var err error + if !pattern.MatchString(name) { + err = errors.New("aws spec must match pattern aws:" + pattern.String()) + } + return name, err +} + +func (t *awsProtocol) NewChunkStore(sp spec.Spec) (chunks.ChunkStore, error) { + parts := strings.SplitN(sp.DatabaseName, "/", 3) // table/bucket/ns + d.PanicIfFalse(len(parts) >= 3) // parse should have ensured this was true + sess := GetAWSSession() + return nbs.NewAWSStore(parts[0], parts[2], parts[1], s3.New(sess), dynamodb.New(sess), 1<<28), nil +} + +func (t *awsProtocol) NewDatabase(sp spec.Spec) (datas.Database, error) { + return datas.NewDatabase(sp.NewChunkStore()), nil +} + +func init() { + spec.ExternalProtocols["aws"] = &awsProtocol{} +} diff --git a/go/spec/aws/aws_test.go b/go/spec/aws/aws_test.go new file mode 100644 index 0000000000..2738c312fc --- /dev/null +++ b/go/spec/aws/aws_test.go @@ -0,0 +1,26 @@ +package aws + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/attic-labs/noms/go/spec/lite" +) + +func TestParse(t *testing.T) { + assert := assert.New(t) + + sp, err := spec.ForDatabase("aws://table/bucket") + assert.Error(err) + assert.Contains(err.Error(), "aws spec must match pattern aws:") + + sp, err = spec.ForDatabase("aws:table/bucket/db") + assert.NoError(err) + assert.Equal("aws", sp.Protocol) + assert.Equal("aws:table/bucket/db", sp.Href()) + + sp, err = spec.ForDatabase("https://localhost/foo/bar/baz") + assert.Error(err) + assert.Contains(err.Error(), "Invalid database protocol https") +} diff --git a/go/spec/http/http.go b/go/spec/http/http.go new file mode 100644 index 0000000000..547d6ea0d2 --- /dev/null +++ b/go/spec/http/http.go @@ -0,0 +1,36 @@ +package http + +import ( + "fmt" + "net/url" + + "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/datas" + "github.com/attic-labs/noms/go/datas/remote" + "github.com/attic-labs/noms/go/spec/lite" +) + +type httpProtocol struct{} + +func (h *httpProtocol) Parse(name string) (string, error) { + u, perr := url.Parse("http:" + name) + if perr != nil { + return "", perr + } else if u.Host == "" { + return "", fmt.Errorf("%s has empty host", name) + } + return name, nil +} + +func (h *httpProtocol) NewChunkStore(sp spec.Spec) (chunks.ChunkStore, error) { + return remote.NewHTTPChunkStore(sp.Href(), sp.Options.Authorization), nil +} + +func (h *httpProtocol) NewDatabase(sp spec.Spec) (datas.Database, error) { + return datas.NewDatabase(sp.NewChunkStore()), nil +} + +func init() { + spec.ExternalProtocols["http"] = &httpProtocol{} + spec.ExternalProtocols["https"] = spec.ExternalProtocols["http"] +} diff --git a/go/spec/http/http_test.go b/go/spec/http/http_test.go new file mode 100644 index 0000000000..277919d1c0 --- /dev/null +++ b/go/spec/http/http_test.go @@ -0,0 +1,27 @@ +package http + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/attic-labs/noms/go/spec/lite" +) + +func TestParse(t *testing.T) { + assert := assert.New(t) + + sp, err := spec.ForDatabase("http://localhost/foo/bar/baz") + assert.NoError(err) + assert.Equal("http", sp.Protocol) + assert.Equal("http://localhost/foo/bar/baz", sp.Href()) + + sp, err = spec.ForDatabase("https://localhost/foo/bar/baz") + assert.NoError(err) + assert.Equal("https", sp.Protocol) + assert.Equal("https://localhost/foo/bar/baz", sp.Href()) + + sp, err = spec.ForDatabase("/var/data") + assert.Error(err) + assert.Contains(err.Error(), "Invalid database protocol nbs in /var/data") +} diff --git a/go/spec/absolute_path.go b/go/spec/lite/absolute_path.go similarity index 100% rename from go/spec/absolute_path.go rename to go/spec/lite/absolute_path.go diff --git a/go/spec/commit_meta.go b/go/spec/lite/commit_meta.go similarity index 100% rename from go/spec/commit_meta.go rename to go/spec/lite/commit_meta.go diff --git a/go/spec/commit_meta_test.go b/go/spec/lite/commit_meta_test.go similarity index 100% rename from go/spec/commit_meta_test.go rename to go/spec/lite/commit_meta_test.go diff --git a/go/spec/lite/lite_test.go b/go/spec/lite/lite_test.go new file mode 100644 index 0000000000..e01273bcee --- /dev/null +++ b/go/spec/lite/lite_test.go @@ -0,0 +1,33 @@ +package spec + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParse(t *testing.T) { + assert := assert.New(t) + + _, err := ForDatabase("http://localhost/foo/bar/baz") + assert.Error(err) + assert.Contains(err.Error(), "Invalid database protocol http in ") + + spec, err := ForDatabase("/var/data") + fmt.Println("Spec:", spec) + assert.Error(err) + assert.Contains(err.Error(), "Invalid database protocol nbs in /var/data") + + _, err = ForDatabase("aws:table/bucket/db") + assert.Error(err) + assert.Contains(err.Error(), "Invalid database protocol aws in") + + spec, err = ForDataset("mem::test") + assert.NoError(err) + defer spec.Close() + assert.Equal("mem", spec.Protocol) + assert.Equal("", spec.DatabaseName) + assert.Equal("test", spec.Path.Dataset) + assert.True(spec.Path.Path.IsEmpty()) +} diff --git a/go/spec/lite/spec.go b/go/spec/lite/spec.go new file mode 100644 index 0000000000..27d653a1bb --- /dev/null +++ b/go/spec/lite/spec.go @@ -0,0 +1,324 @@ +// Copyright 2016 Attic Labs, Inc. All rights reserved. +// Licensed under the Apache License, version 2.0: +// http://www.apache.org/licenses/LICENSE-2.0 + +// Package spec provides builders and parsers for spelling Noms databases, +// datasets and values. +package spec + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/d" + "github.com/attic-labs/noms/go/datas" + "github.com/attic-labs/noms/go/types" +) + +const Separator = "::" + +var datasetRe = regexp.MustCompile("^" + datas.DatasetRe.String() + "$") + +type ProtocolImpl interface { + NewChunkStore(sp Spec) (chunks.ChunkStore, error) + NewDatabase(sp Spec) (datas.Database, error) +} + +type protocolParser interface { + Parse(name string) (string, error) + ProtocolImpl +} + +var ExternalProtocols = map[string]ProtocolImpl{} + +// SpecOptions customize Spec behavior. +type SpecOptions struct { + // Authorization token for requests. For example, if the database is HTTP + // this will used for an `Authorization: Bearer ${authorization}` header. + Authorization string +} + +// Spec locates a Noms database, dataset, or value globally. Spec caches +// its database instance so it therefore does not reflect new commits in +// the db, by (legacy) design. +type Spec struct { + // Protocol is one of "mem", "aws", "ndb", "http", or "https". + Protocol string + + // DatabaseName is the name of the Spec's database, which is the string after + // "protocol:". http/https specs include their leading "//" characters. + DatabaseName string + + // Options are the SpecOptions that the Spec was constructed with. + Options SpecOptions + + // Path is nil unless the spec was created with ForPath. + Path AbsolutePath + + // db is lazily created, so it needs to be a pointer to a Database. + db *datas.Database +} + +func newSpec(dbSpec string, opts SpecOptions) (Spec, error) { + protocol, dbName, err := parseDatabaseSpec(dbSpec) + if err != nil { + return Spec{}, err + } + + return Spec{ + Protocol: protocol, + DatabaseName: dbName, + Options: opts, + db: new(datas.Database), + }, nil +} + +// ForDatabase parses a spec for a Database. +func ForDatabase(spec string) (Spec, error) { + return ForDatabaseOpts(spec, SpecOptions{}) +} + +// ForDatabaseOpts parses a spec for a Database. +func ForDatabaseOpts(spec string, opts SpecOptions) (Spec, error) { + return newSpec(spec, opts) +} + +// ForDataset parses a spec for a Dataset. +func ForDataset(spec string) (Spec, error) { + return ForDatasetOpts(spec, SpecOptions{}) +} + +// ForDatasetOpts parses a spec for a Dataset. +func ForDatasetOpts(spec string, opts SpecOptions) (Spec, error) { + dbSpec, pathStr, err := splitDatabaseSpec(spec) + if err != nil { + return Spec{}, err + } + + sp, err := newSpec(dbSpec, opts) + if err != nil { + return Spec{}, err + } + + path, err := NewAbsolutePath(pathStr) + if err != nil { + return Spec{}, err + } + + if path.Dataset == "" { + return Spec{}, errors.New("dataset name required for dataset spec") + } + + if !path.Path.IsEmpty() { + return Spec{}, errors.New("path is not allowed for dataset spec") + } + + sp.Path = path + return sp, nil +} + +// ForPath parses a spec for a path to a Value. +func ForPath(spec string) (Spec, error) { + return ForPathOpts(spec, SpecOptions{}) +} + +// ForPathOpts parses a spec for a path to a Value. +func ForPathOpts(spec string, opts SpecOptions) (Spec, error) { + dbSpec, pathStr, err := splitDatabaseSpec(spec) + if err != nil { + return Spec{}, err + } + + var path AbsolutePath + if pathStr != "" { + path, err = NewAbsolutePath(pathStr) + if err != nil { + return Spec{}, err + } + } + + sp, err := newSpec(dbSpec, opts) + if err != nil { + return Spec{}, err + } + + sp.Path = path + return sp, nil +} + +func (sp Spec) String() string { + s := sp.Protocol + if s != "mem" { + s += ":" + sp.DatabaseName + } + p := sp.Path.String() + if p != "" { + s += Separator + p + } + return s +} + +// GetDatabase returns the Database instance that this Spec's DatabaseName +// describes. The same Database instance is returned every time, unless Close +// is called. If the Spec is closed, it is re-opened with a new Database. +func (sp Spec) GetDatabase() datas.Database { + if *sp.db == nil { + *sp.db = sp.createDatabase() + } + return *sp.db +} + +// GetDataset returns the current Dataset instance for this Spec's Database. +// GetDataset is live, so if Commit is called on this Spec's Database later, a +// new up-to-date Dataset will returned on the next call to GetDataset. If +// this is not a Dataset spec, returns nil. +func (sp Spec) GetDataset() (ds datas.Dataset) { + if sp.Path.Dataset != "" { + ds = sp.GetDatabase().GetDataset(sp.Path.Dataset) + } + return +} + +// GetValue returns the Value at this Spec's Path within its Database, or nil +// if this isn't a Path Spec or if that path isn't found. +func (sp Spec) GetValue() (val types.Value) { + if !sp.Path.IsEmpty() { + val = sp.Path.Resolve(sp.GetDatabase()) + } + return +} + +// Href treats the Protocol and DatabaseName as a URL, and returns its href. +// For example, the spec http://example.com/path::ds returns +// "http://example.com/path". If the Protocol is not "http" or "http", returns +// an empty string. +func (sp Spec) Href() string { + switch proto := sp.Protocol; proto { + case "http", "https", "aws": + return proto + ":" + sp.DatabaseName + default: + return "" + } +} + +// Pin returns a Spec in which the dataset component, if any, has been replaced +// with the hash of the HEAD of that dataset. This "pins" the path to the state +// of the database at the current moment in time. Returns itself if the +// PathSpec is already "pinned". +func (sp Spec) Pin() (Spec, bool) { + var ds datas.Dataset + + if !sp.Path.IsEmpty() { + if !sp.Path.Hash.IsEmpty() { + // Spec is already pinned. + return sp, true + } + + ds = sp.GetDatabase().GetDataset(sp.Path.Dataset) + } else { + ds = sp.GetDataset() + } + + commit, ok := ds.MaybeHead() + if !ok { + return Spec{}, false + } + + r := sp + r.Path.Hash = commit.Hash() + r.Path.Dataset = "" + + return r, true +} + +func (sp Spec) Close() error { + db := *sp.db + if db == nil { + return nil + } + + *sp.db = nil + return db.Close() +} + +func (sp Spec) createDatabase() datas.Database { + switch sp.Protocol { + case "mem": + return datas.NewDatabase(sp.NewChunkStore()) + default: + impl, ok := ExternalProtocols[sp.Protocol] + if !ok { + d.PanicIfError(fmt.Errorf("Unknown protocol: %s", sp.Protocol)) + } + r, err := impl.NewDatabase(sp) + d.PanicIfError(err) + return r + } +} + +// NewChunkStore returns a new ChunkStore instance that this Spec's +// DatabaseName describes. It's unusual to call this method, GetDatabase is +// more useful. +func (sp Spec) NewChunkStore() chunks.ChunkStore { + if sp.Protocol == "mem" { + storage := &chunks.MemoryStorage{} + return storage.NewView() + } + impl, ok := ExternalProtocols[sp.Protocol] + if !ok { + d.PanicIfError(fmt.Errorf("Unknown protocol: %s", sp.Protocol)) + } + r, err := impl.NewChunkStore(sp) + d.PanicIfError(err) + return r +} + +func parseDatabaseSpec(spec string) (protocol, name string, err error) { + if len(spec) == 0 { + err = fmt.Errorf("Empty spec") + return + } + + parts := strings.SplitN(spec, ":", 2) // [protocol] [, path]? + + // If there was no ":" then this is either a mem spec, or a filesystem path. + // This is ambiguous if the file system path is "mem" but that just means the + // path needs to be explicitly "nbs:mem". + if len(parts) == 1 { + if spec == "mem" { + protocol = "mem" + return + } else { + parts = []string{"nbs", spec} + } + } + + if _, ok := ExternalProtocols[parts[0]]; ok { + protocol, name = parts[0], parts[1] + parser, ok := ExternalProtocols[protocol].(protocolParser) + if ok { + name, err = parser.Parse(name) + } + return + } + + if parts[0] == "mem" { + err = fmt.Errorf(`In-memory database must be specified as "mem", not "mem:"`) + } else { + + err = fmt.Errorf("Invalid database protocol %s in %s", parts[0], spec) + } + return +} + +func splitDatabaseSpec(spec string) (string, string, error) { + lastIdx := strings.LastIndex(spec, Separator) + if lastIdx == -1 { + return "", "", fmt.Errorf("Missing %s after database in %s", Separator, spec) + } + + return spec[:lastIdx], spec[lastIdx+len(Separator):], nil +} diff --git a/go/spec/util.go b/go/spec/lite/util.go similarity index 100% rename from go/spec/util.go rename to go/spec/lite/util.go diff --git a/go/spec/nbs/nbs.go b/go/spec/nbs/nbs.go new file mode 100644 index 0000000000..6551e09da2 --- /dev/null +++ b/go/spec/nbs/nbs.go @@ -0,0 +1,25 @@ +package nbs + +import ( + "os" + + "github.com/attic-labs/noms/go/chunks" + "github.com/attic-labs/noms/go/datas" + "github.com/attic-labs/noms/go/nbs" + "github.com/attic-labs/noms/go/spec/lite" +) + +type nbsProtocol struct{} + +func (n *nbsProtocol) NewChunkStore(sp spec.Spec) (chunks.ChunkStore, error) { + os.MkdirAll(sp.DatabaseName, 0777) + return nbs.NewLocalStore(sp.DatabaseName, 1<<28), nil +} + +func (n *nbsProtocol) NewDatabase(sp spec.Spec) (datas.Database, error) { + return datas.NewDatabase(sp.NewChunkStore()), nil +} + +func init() { + spec.ExternalProtocols["nbs"] = &nbsProtocol{} +} diff --git a/go/spec/nbs/nbs_test.go b/go/spec/nbs/nbs_test.go new file mode 100644 index 0000000000..9848abb246 --- /dev/null +++ b/go/spec/nbs/nbs_test.go @@ -0,0 +1,27 @@ +package nbs + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/attic-labs/noms/go/spec/lite" +) + +func TestParse(t *testing.T) { + assert := assert.New(t) + + sp, err := spec.ForDatabase("/var/data") + assert.NoError(err) + assert.Equal("nbs", sp.Protocol) + assert.Equal("/var/data", sp.DatabaseName) + + sp, err = spec.ForDatabase("nbs:/var/data") + assert.NoError(err) + assert.Equal("nbs", sp.Protocol) + assert.Equal("/var/data", sp.DatabaseName) + + sp, err = spec.ForDatabase("http://localhost/foo/bar/baz") + assert.Error(err) + assert.Contains(err.Error(), "Invalid database protocol http") +} diff --git a/go/spec/spec.go b/go/spec/spec.go index 9a16437502..f51b58f6a0 100644 --- a/go/spec/spec.go +++ b/go/spec/spec.go @@ -1,362 +1,72 @@ -// Copyright 2016 Attic Labs, Inc. All rights reserved. -// Licensed under the Apache License, version 2.0: -// http://www.apache.org/licenses/LICENSE-2.0 - -// Package spec provides builders and parsers for spelling Noms databases, -// datasets and values. package spec import ( - "errors" - "fmt" - "net/url" - "os" - "regexp" - "strings" - - "github.com/attic-labs/noms/go/chunks" - "github.com/attic-labs/noms/go/d" "github.com/attic-labs/noms/go/datas" - "github.com/attic-labs/noms/go/nbs" + "github.com/attic-labs/noms/go/hash" + "github.com/attic-labs/noms/go/spec/lite" "github.com/attic-labs/noms/go/types" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/dynamodb" - "github.com/aws/aws-sdk-go/service/s3" -) - -const Separator = "::" - -var datasetRe = regexp.MustCompile("^" + datas.DatasetRe.String() + "$") - -var GetAWSSession func() *session.Session = func() *session.Session { - return session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2"))) -} - -type ProtocolImpl interface { - NewChunkStore(sp Spec) (chunks.ChunkStore, error) - NewDatabase(sp Spec) (datas.Database, error) -} - -var ExternalProtocols = map[string]ProtocolImpl{} - -// SpecOptions customize Spec behavior. -type SpecOptions struct { - // Authorization token for requests. For example, if the database is HTTP - // this will used for an `Authorization: Bearer ${authorization}` header. - Authorization string -} -// Spec locates a Noms database, dataset, or value globally. Spec caches -// its database instance so it therefore does not reflect new commits in -// the db, by (legacy) design. -type Spec struct { - // Protocol is one of "mem", "ldb", "http", or "https". - Protocol string - - // DatabaseName is the name of the Spec's database, which is the string after - // "protocol:". http/https specs include their leading "//" characters. - DatabaseName string - - // Options are the SpecOptions that the Spec was constructed with. - Options SpecOptions - - // Path is nil unless the spec was created with ForPath. - Path AbsolutePath + "github.com/attic-labs/noms/go/spec/aws" + _ "github.com/attic-labs/noms/go/spec/http" + _ "github.com/attic-labs/noms/go/spec/nbs" +) - // db is lazily created, so it needs to be a pointer to a Database. - db *datas.Database -} +type ( + Spec = spec.Spec + SpecOptions = spec.SpecOptions + AbsolutePath = spec.AbsolutePath +) -func newSpec(dbSpec string, opts SpecOptions) (Spec, error) { - protocol, dbName, err := parseDatabaseSpec(dbSpec) - if err != nil { - return Spec{}, err - } +var ( + ExternalProtocols = spec.ExternalProtocols + GetAWSSession = aws.GetAWSSession +) - return Spec{ - Protocol: protocol, - DatabaseName: dbName, - Options: opts, - db: new(datas.Database), - }, nil -} +const ( + Separator = spec.Separator + CommitMetaDateFormat = spec.CommitMetaDateFormat +) // ForDatabase parses a spec for a Database. -func ForDatabase(spec string) (Spec, error) { - return ForDatabaseOpts(spec, SpecOptions{}) -} - -// ForDatabaseOpts parses a spec for a Database. -func ForDatabaseOpts(spec string, opts SpecOptions) (Spec, error) { - return newSpec(spec, opts) -} - -// ForDataset parses a spec for a Dataset. -func ForDataset(spec string) (Spec, error) { - return ForDatasetOpts(spec, SpecOptions{}) -} - -// ForDatasetOpts parses a spec for a Dataset. -func ForDatasetOpts(spec string, opts SpecOptions) (Spec, error) { - dbSpec, pathStr, err := splitDatabaseSpec(spec) - if err != nil { - return Spec{}, err - } - - sp, err := newSpec(dbSpec, opts) - if err != nil { - return Spec{}, err - } - - path, err := NewAbsolutePath(pathStr) - if err != nil { - return Spec{}, err - } - - if path.Dataset == "" { - return Spec{}, errors.New("dataset name required for dataset spec") - } - - if !path.Path.IsEmpty() { - return Spec{}, errors.New("path is not allowed for dataset spec") - } - - sp.Path = path - return sp, nil +func ForDatabase(sp string) (Spec, error) { + return spec.ForDatabase(sp) } // ForPath parses a spec for a path to a Value. -func ForPath(spec string) (Spec, error) { - return ForPathOpts(spec, SpecOptions{}) -} - -// ForPathOpts parses a spec for a path to a Value. -func ForPathOpts(spec string, opts SpecOptions) (Spec, error) { - dbSpec, pathStr, err := splitDatabaseSpec(spec) - if err != nil { - return Spec{}, err - } - - var path AbsolutePath - if pathStr != "" { - path, err = NewAbsolutePath(pathStr) - if err != nil { - return Spec{}, err - } - } - - sp, err := newSpec(dbSpec, opts) - if err != nil { - return Spec{}, err - } - - sp.Path = path - return sp, nil +func ForPath(sp string) (Spec, error) { + return spec.ForPathOpts(sp, SpecOptions{}) } -func (sp Spec) String() string { - s := sp.Protocol - if s != "mem" { - s += ":" + sp.DatabaseName - } - p := sp.Path.String() - if p != "" { - s += Separator + p - } - return s +// ForDatabaseOpts parses a spec for a Database. +func ForDatabaseOpts(sp string, opts SpecOptions) (Spec, error) { + return spec.ForDatabaseOpts(sp, opts) } -// GetDatabase returns the Database instance that this Spec's DatabaseName -// describes. The same Database instance is returned every time, unless Close -// is called. If the Spec is closed, it is re-opened with a new Database. -func (sp Spec) GetDatabase() datas.Database { - if *sp.db == nil { - *sp.db = sp.createDatabase() - } - return *sp.db +// ForDataset parses a spec for a Dataset. +func ForDataset(sp string) (Spec, error) { + return spec.ForDataset(sp) } -// GetDataset returns the current Dataset instance for this Spec's Database. -// GetDataset is live, so if Commit is called on this Spec's Database later, a -// new up-to-date Dataset will returned on the next call to GetDataset. If -// this is not a Dataset spec, returns nil. -func (sp Spec) GetDataset() (ds datas.Dataset) { - if sp.Path.Dataset != "" { - ds = sp.GetDatabase().GetDataset(sp.Path.Dataset) - } - return +func NewAbsolutePath(str string) (AbsolutePath, error) { + return spec.NewAbsolutePath(str) } -// GetValue returns the Value at this Spec's Path within its Database, or nil -// if this isn't a Path Spec or if that path isn't found. -func (sp Spec) GetValue() (val types.Value) { - if !sp.Path.IsEmpty() { - val = sp.Path.Resolve(sp.GetDatabase()) - } - return +func ReadAbsolutePaths(db datas.Database, paths ...string) ([]types.Value, error) { + return spec.ReadAbsolutePaths(db, paths...) } -// Href treats the Protocol and DatabaseName as a URL, and returns its href. -// For example, the spec http://example.com/path::ds returns -// "http://example.com/path". If the Protocol is not "http" or "http", returns -// an empty string. -func (sp Spec) Href() string { - switch proto := sp.Protocol; proto { - case "http", "https", "aws": - return proto + ":" + sp.DatabaseName - default: - return "" - } +func CreateDatabaseSpecString(protocol, db string) string { + return spec.CreateDatabaseSpecString(protocol, db) } -// Pin returns a Spec in which the dataset component, if any, has been replaced -// with the hash of the HEAD of that dataset. This "pins" the path to the state -// of the database at the current moment in time. Returns itself if the -// PathSpec is already "pinned". -func (sp Spec) Pin() (Spec, bool) { - var ds datas.Dataset - - if !sp.Path.IsEmpty() { - if !sp.Path.Hash.IsEmpty() { - // Spec is already pinned. - return sp, true - } - - ds = sp.GetDatabase().GetDataset(sp.Path.Dataset) - } else { - ds = sp.GetDataset() - } - - commit, ok := ds.MaybeHead() - if !ok { - return Spec{}, false - } - - r := sp - r.Path.Hash = commit.Hash() - r.Path.Dataset = "" - - return r, true +func CreateValueSpecString(protocol, db, path string) string { + return spec.CreateValueSpecString(protocol, db, path) } -func (sp Spec) Close() error { - db := *sp.db - if db == nil { - return nil - } - - *sp.db = nil - return db.Close() +func CreateHashSpecString(protocol, db string, h hash.Hash) string { + return spec.CreateHashSpecString(protocol, db, h) } -func (sp Spec) createDatabase() datas.Database { - switch sp.Protocol { - case "http", "https", "aws", "nbs", "mem": - return datas.NewDatabase(sp.NewChunkStore()) - default: - impl, ok := ExternalProtocols[sp.Protocol] - if !ok { - d.PanicIfError(fmt.Errorf("Unknown protocol: %s", sp.Protocol)) - } - r, err := impl.NewDatabase(sp) - d.PanicIfError(err) - return r - } -} - -// NewChunkStore returns a new ChunkStore instance that this Spec's -// DatabaseName describes. It's unusual to call this method, GetDatabase is -// more useful. -func (sp Spec) NewChunkStore() chunks.ChunkStore { - switch sp.Protocol { - case "http", "https": - return datas.NewHTTPChunkStore(sp.Href(), sp.Options.Authorization) - case "aws": - parts := strings.SplitN(sp.DatabaseName, "/", 3) // table/bucket/ns - d.PanicIfFalse(len(parts) >= 3) // parse should have ensured this was true - sess := GetAWSSession() - return nbs.NewAWSStore(parts[0], parts[2], parts[1], s3.New(sess), dynamodb.New(sess), 1<<28) - case "nbs": - os.MkdirAll(sp.DatabaseName, 0777) - return nbs.NewLocalStore(sp.DatabaseName, 1<<28) - case "mem": - storage := &chunks.MemoryStorage{} - return storage.NewView() - default: - impl, ok := ExternalProtocols[sp.Protocol] - if !ok { - d.PanicIfError(fmt.Errorf("Unknown protocol: %s", sp.Protocol)) - } - r, err := impl.NewChunkStore(sp) - d.PanicIfError(err) - return r - } -} - -func parseDatabaseSpec(spec string) (protocol, name string, err error) { - if len(spec) == 0 { - err = fmt.Errorf("Empty spec") - return - } - - parts := strings.SplitN(spec, ":", 2) // [protocol] [, path]? - - // If there was no ":" then this is either a mem spec, or a filesystem path. - // This is ambiguous if the file system path is "mem" but that just means the - // path needs to be explicitly "nbs:mem". - if len(parts) == 1 { - if spec == "mem" { - protocol = "mem" - } else { - protocol, name = "nbs", spec - } - return - } - - if _, ok := ExternalProtocols[parts[0]]; ok { - fmt.Println("found external spec", parts[0]) - protocol, name = parts[0], parts[1] - return - } - - switch parts[0] { - case "nbs": - protocol, name = parts[0], parts[1] - - case "aws": - p, n := parts[0], parts[1] - pattern := regexp.MustCompile("^[^/]+/[^/]+/.*$") - if !pattern.MatchString(n) { - err = errors.New("aws spec must match pattern aws:" + pattern.String()) - } - protocol, name = p, n - return - - case "http", "https": - u, perr := url.Parse(spec) - if perr != nil { - err = perr - } else if u.Host == "" { - err = fmt.Errorf("%s has empty host", spec) - } else { - protocol, name = parts[0], parts[1] - } - - case "mem": - err = fmt.Errorf(`In-memory database must be specified as "mem", not "mem:"`) - - default: - err = fmt.Errorf("Invalid database protocol %s in %s", protocol, spec) - } - return -} - -func splitDatabaseSpec(spec string) (string, string, error) { - lastIdx := strings.LastIndex(spec, Separator) - if lastIdx == -1 { - return "", "", fmt.Errorf("Missing %s after database in %s", Separator, spec) - } - - return spec[:lastIdx], spec[lastIdx+len(Separator):], nil +func CreateCommitMetaStruct(db datas.Database, date, message string, keyValueStrings map[string]string, keyValuePaths map[string]types.Value) (types.Struct, error) { + return spec.CreateCommitMetaStruct(db, date, message, keyValueStrings, keyValuePaths) } diff --git a/go/types/encode_human_readable.go b/go/types/encode_human_readable.go index 740f9fa7e0..d5ad31b5cf 100644 --- a/go/types/encode_human_readable.go +++ b/go/types/encode_human_readable.go @@ -399,7 +399,9 @@ func encodedValueFormatMaxLines(v Value, floatFormat byte, maxLines uint32) stri w := &hrsWriter{w: mlw, floatFormat: floatFormat} w.Write(v) if w.err != nil { - d.Chk.IsType(writers.MaxLinesError{}, w.err, "Unexpected error: %s", w.err) + if _, ok := w.err.(writers.MaxLinesError); !ok { + d.Chk.Fail("Unexpected error: %s", w.err) + } } return buf.String() } diff --git a/go/types/graph_builder.go b/go/types/graph_builder.go index 0e1ae9fddd..721421ec16 100644 --- a/go/types/graph_builder.go +++ b/go/types/graph_builder.go @@ -36,6 +36,8 @@ // calls to the mutation operations have completed before Build() is invoked. // +// +build !js,!wasm + package types import ( diff --git a/go/types/opcache.go b/go/types/opcache.go index 188b4499a4..c9a5f4dd07 100644 --- a/go/types/opcache.go +++ b/go/types/opcache.go @@ -59,6 +59,8 @@ // ldbKey. The value being appended to the list. // +// +build !js,!wasm + package types import ( diff --git a/go/types/opcache_compare.go b/go/types/opcache_compare.go index db11ae0ec0..278adbdba0 100644 --- a/go/types/opcache_compare.go +++ b/go/types/opcache_compare.go @@ -2,6 +2,8 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 +// +build !js,!wasm + package types import ( diff --git a/go/types/opcache_test.go b/go/types/opcache_test.go index a77004084a..47cf6afe1b 100644 --- a/go/types/opcache_test.go +++ b/go/types/opcache_test.go @@ -9,7 +9,6 @@ import ( "sort" "testing" - "github.com/attic-labs/noms/go/d" "github.com/stretchr/testify/suite" ) @@ -57,8 +56,8 @@ func (suite *OpCacheSuite) TestMapSet() { defer iter.Release() for iter.Next() { keys, kind, item := iter.GraphOp() - d.Chk.Empty(keys) - d.Chk.Equal(MapKind, kind) + suite.Empty(keys) + suite.Equal(MapKind, kind) iterated = append(iterated, item.(mapEntry)) } suite.True(entries.Equals(iterated)) @@ -95,8 +94,8 @@ func (suite *OpCacheSuite) TestSetInsert() { defer iter.Release() for iter.Next() { keys, kind, item := iter.GraphOp() - d.Chk.Empty(keys) - d.Chk.Equal(SetKind, kind) + suite.Empty(keys) + suite.Equal(SetKind, kind) iterated = append(iterated, item.(Value)) } suite.True(entries.Equals(iterated)) @@ -132,8 +131,8 @@ func (suite *OpCacheSuite) TestListAppend() { defer iter.Release() for iter.Next() { keys, kind, item := iter.GraphOp() - d.Chk.Empty(keys) - d.Chk.Equal(ListKind, kind) + suite.Empty(keys) + suite.Equal(ListKind, kind) iterated = append(iterated, item.(Value)) } suite.True(entries.Equals(iterated)) diff --git a/go/types/subtype_test.go b/go/types/subtype_test.go index 9441e2b4d8..47d0a5d7f4 100644 --- a/go/types/subtype_test.go +++ b/go/types/subtype_test.go @@ -9,7 +9,6 @@ import ( "strings" "testing" - "github.com/attic-labs/noms/go/d" "github.com/stretchr/testify/assert" ) @@ -461,10 +460,10 @@ func makeTestStructTypeFromFieldNames(s string) *Type { return MakeStructType("", fields...) } -func makeTestStructFromFieldNames(s string) Struct { +func makeTestStructFromFieldNames(assert *assert.Assertions, s string) Struct { t := makeTestStructTypeFromFieldNames(s) fields := t.Desc.(StructDesc).fields - d.Chk.NotEmpty(fields) + assert.NotEmpty(fields) fieldNames := make([]string, len(fields)) for i, field := range fields { @@ -805,7 +804,7 @@ func TestIsValueSubtypeOfDetails(tt *testing.T) { a := assert.New(tt) test := func(vString, tString string, exp1, exp2 bool) { - v := makeTestStructFromFieldNames(vString) + v := makeTestStructFromFieldNames(a, vString) t := makeTestStructTypeFromFieldNames(tString) isSub, hasExtra := IsValueSubtypeOfDetails(v, t) a.Equal(exp1, isSub, "expected %t for IsSub, received: %t", exp1, isSub) diff --git a/go/util/verbose/verbose.go b/go/util/verbose/verbose.go index 0119b28646..178e2bc8e3 100644 --- a/go/util/verbose/verbose.go +++ b/go/util/verbose/verbose.go @@ -6,8 +6,6 @@ package verbose import ( "log" - - "github.com/attic-labs/kingpin" ) var ( @@ -15,15 +13,6 @@ var ( quiet bool ) -// RegisterVerboseFlags registers -v|--verbose flags for general usage -func RegisterVerboseFlags(app *kingpin.Application) { - // Must reset globals because under test this can get called multiple times. - verbose = false - quiet = false - app.Flag("verbose", "show more").Short('v').BoolVar(&verbose) - app.Flag("quite", "show less").Short('q').BoolVar(&quiet) -} - // Verbose returns True if the verbose flag was set func Verbose() bool { return verbose diff --git a/go/util/verbose/verbose_test.go b/go/util/verbose/verbose_test.go new file mode 100644 index 0000000000..914b3c8609 --- /dev/null +++ b/go/util/verbose/verbose_test.go @@ -0,0 +1,42 @@ +package verbose_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/attic-labs/kingpin" + + "github.com/attic-labs/noms/go/util/verbose" + "github.com/attic-labs/noms/go/util/verbose/verboseflags" +) + +func TestVerbose(t *testing.T) { + app := kingpin.New("app", "") + for _, tt := range []struct { + args string + verbose bool + quiet bool + }{ + {args: "-v", verbose: true}, + {args: "--verbose", verbose: true}, + {args: "-q", quiet: true}, + {args: "--quiet", quiet: true}, + {args: "-vq", verbose: true, quiet: true}, + {args: "--verbose --quiet", verbose: true, quiet: true}, + } { + t.Run(tt.args, func(t *testing.T) { + assert := assert.New(t) + + verboseflags.Register(app) + assert.False(verbose.Verbose()) + assert.False(verbose.Quiet()) + res, err := app.Parse(strings.Split(tt.args, " ")) + assert.Empty(res) + assert.NoError(err) + assert.Equal(tt.verbose, verbose.Verbose()) + assert.Equal(tt.quiet, verbose.Quiet()) + }) + } +} diff --git a/go/util/verbose/verboseflags/register.go b/go/util/verbose/verboseflags/register.go new file mode 100644 index 0000000000..274100d9f6 --- /dev/null +++ b/go/util/verbose/verboseflags/register.go @@ -0,0 +1,24 @@ +package verboseflags + +import ( + "github.com/attic-labs/kingpin" + + "github.com/attic-labs/noms/go/util/verbose" +) + +// Register registers -v|--verbose flags for general usage +func Register(app *kingpin.Application) { + // Must reset globals because under test this can get called multiple times. + verbose.SetVerbose(false) + verbose.SetQuiet(false) + loud := false + quiet := false + app.Flag("verbose", "show more").Short('v').Action(func(ctx *kingpin.ParseContext) error { + verbose.SetVerbose(loud) + return nil + }).BoolVar(&loud) + app.Flag("quiet", "show less").Short('q').Action(func(ctx *kingpin.ParseContext) error { + verbose.SetQuiet(quiet) + return nil + }).BoolVar(&quiet) +} diff --git a/samples/go/counter/counter.go b/samples/go/counter/counter.go index 117af3b4a5..3aca9cd3a7 100644 --- a/samples/go/counter/counter.go +++ b/samples/go/counter/counter.go @@ -12,13 +12,13 @@ import ( "github.com/attic-labs/noms/go/config" "github.com/attic-labs/noms/go/types" - "github.com/attic-labs/noms/go/util/verbose" + "github.com/attic-labs/noms/go/util/verbose/verboseflags" ) func main() { app := kingpin.New("counter", "") dsStr := app.Arg("ds", "dataset to count in").Required().String() - verbose.RegisterVerboseFlags(app) + verboseflags.Register(app) kingpin.MustParse(app.Parse(os.Args[1:])) cfg := config.NewResolver() diff --git a/samples/go/csv/csv-export/exporter.go b/samples/go/csv/csv-export/exporter.go index fe2cd6892a..48f3832117 100644 --- a/samples/go/csv/csv-export/exporter.go +++ b/samples/go/csv/csv-export/exporter.go @@ -14,7 +14,7 @@ import ( "github.com/attic-labs/noms/go/d" "github.com/attic-labs/noms/go/types" "github.com/attic-labs/noms/go/util/profile" - "github.com/attic-labs/noms/go/util/verbose" + "github.com/attic-labs/noms/go/util/verbose/verboseflags" "github.com/attic-labs/noms/samples/go/csv" ) @@ -26,7 +26,7 @@ func main() { delimiter := app.Flag("delimiter", "field delimiter for csv file, must be exactly one character long.").Default(",").String() dataset := app.Arg("dataset", "dataset to export").Required().String() - verbose.RegisterVerboseFlags(app) + verboseflags.Register(app) profile.RegisterProfileFlags(app) kingpin.MustParse(app.Parse(os.Args[1:])) diff --git a/samples/go/csv/csv-import/importer.go b/samples/go/csv/csv-import/importer.go index e1ee5f561a..58e994e3ed 100644 --- a/samples/go/csv/csv-import/importer.go +++ b/samples/go/csv/csv-import/importer.go @@ -24,7 +24,7 @@ import ( "github.com/attic-labs/noms/go/util/profile" "github.com/attic-labs/noms/go/util/progressreader" "github.com/attic-labs/noms/go/util/status" - "github.com/attic-labs/noms/go/util/verbose" + "github.com/attic-labs/noms/go/util/verbose/verboseflags" "github.com/attic-labs/noms/samples/go/csv" ) @@ -54,7 +54,7 @@ func main() { dataset := app.Arg("dataset", "datset to write to").Required().String() csvFile := app.Arg("csvfile", "csv file to import").String() - verbose.RegisterVerboseFlags(app) + verboseflags.Register(app) profile.RegisterProfileFlags(app) kingpin.MustParse(app.Parse(os.Args[1:])) diff --git a/samples/go/hr/main.go b/samples/go/hr/main.go index 54e07d5290..a38bc2048a 100644 --- a/samples/go/hr/main.go +++ b/samples/go/hr/main.go @@ -14,7 +14,7 @@ import ( "github.com/attic-labs/noms/go/datas" "github.com/attic-labs/noms/go/marshal" "github.com/attic-labs/noms/go/types" - "github.com/attic-labs/noms/go/util/verbose" + "github.com/attic-labs/noms/go/util/verbose/verboseflags" ) func main() { @@ -28,7 +28,7 @@ func main() { app.Command("list-persons", "list current persons") - verbose.RegisterVerboseFlags(app) + verboseflags.Register(app) cmd := kingpin.MustParse(app.Parse(os.Args[1:])) cfg := config.NewResolver() db, ds, err := cfg.GetDataset(*dsStr) diff --git a/samples/go/nomdex/nomdex.go b/samples/go/nomdex/nomdex.go index f591fd85f3..4ad6de0a25 100644 --- a/samples/go/nomdex/nomdex.go +++ b/samples/go/nomdex/nomdex.go @@ -12,13 +12,13 @@ import ( "github.com/attic-labs/noms/go/d" "github.com/attic-labs/noms/go/util/profile" - "github.com/attic-labs/noms/go/util/verbose" + "github.com/attic-labs/noms/go/util/verbose/verboseflags" ) func main() { registerUpdate() registerFind() - verbose.RegisterVerboseFlags(kingpin.CommandLine) + verboseflags.Register(kingpin.CommandLine) profile.RegisterProfileFlags(kingpin.CommandLine) switch kingpin.Parse() { diff --git a/samples/go/nomsfs/nomsfs.go b/samples/go/nomsfs/nomsfs.go index e2a85c9390..a94df1635f 100644 --- a/samples/go/nomsfs/nomsfs.go +++ b/samples/go/nomsfs/nomsfs.go @@ -14,6 +14,7 @@ import ( "os" "os/signal" "path" + "reflect" "runtime" "strings" "sync" @@ -394,7 +395,7 @@ func (fs *nomsFS) Readlink(path string, context *fuse.Context) (string, fuse.Sta } inode := np.inode - d.Chk.Equal(nodeType(inode), "Symlink") + d.Chk.StringEqual(nodeType(inode), "Symlink") link := inode.Get("contents") return string(link.(types.Struct).Get("targetPath").(types.String)), fuse.OK @@ -405,13 +406,13 @@ func (fs *nomsFS) Unlink(path string, context *fuse.Context) fuse.Status { // Since we don't support hard links we don't need to worry about checking the link count. return fs.removeCommon(path, func(inode types.Value) { - d.Chk.NotEqual(nodeType(inode), "Directory") + d.Chk.StringNotEqual(nodeType(inode), "Directory") }) } func (fs *nomsFS) Rmdir(path string, context *fuse.Context) (code fuse.Status) { return fs.removeCommon(path, func(inode types.Value) { - d.Chk.Equal(nodeType(inode), "Directory") + d.Chk.StringEqual(nodeType(inode), "Directory") }) } @@ -448,7 +449,7 @@ func (nfile nomsFile) Read(dest []byte, off int64) (fuse.ReadResult, fuse.Status file := nfile.node.inode.Get("contents") - d.Chk.Equal(nodeType(nfile.node.inode), "File") + d.Chk.StringEqual(nodeType(nfile.node.inode), "File") ref := file.(types.Struct).Get("data").(types.Ref) blob := ref.TargetValue(nfile.fs.db).(types.Blob) @@ -468,7 +469,7 @@ func (nfile nomsFile) Write(data []byte, off int64) (uint32, fuse.Status) { defer nfile.node.nLock.Unlock() inode := nfile.node.inode - d.Chk.Equal(nodeType(inode), "File") + d.Chk.StringEqual(nodeType(inode), "File") attr := inode.Get("attr").(types.Struct) file := inode.Get("contents").(types.Struct) @@ -541,13 +542,18 @@ func nodeType(inode types.Value) string { func (fs *nomsFS) getNode(inode types.Struct, name string, parent *nNode) *nNode { // The parent has to be a directory. if parent != nil { - d.Chk.Equal("Directory", nodeType(parent.inode)) + d.Chk.StringEqual("Directory", nodeType(parent.inode)) } np, ok := fs.nodes[inode.Hash()] if ok { - d.Chk.Equal(np.parent, parent) - d.Chk.Equal(np.name, name) + if ((np.parent == nil || parent == nil) && np.parent != parent) || + !reflect.DeepEqual(np.parent, parent) { + d.Chk.Fail(fmt.Sprintf("Not equal: \n"+ + "expected: %#v\n"+ + "actual : %#v", np.parent, parent)) + } + d.Chk.StringEqual(np.name, name) } else { np = &nNode{ nLock: &sync.Mutex{}, @@ -617,7 +623,7 @@ func (fs *nomsFS) getPathComponents(components []string) (*nNode, fuse.Status) { np := fs.getNode(inode, "", nil) for _, component := range components { - d.Chk.NotEqual(component, "") + d.Chk.StringNotEqual(component, "") contents := inode.Get("contents") if types.TypeOf(contents).Desc.(types.StructDesc).Name != "Directory" { diff --git a/samples/go/xml-import/xml_importer.go b/samples/go/xml-import/xml_importer.go index dff21f2339..021571dadd 100644 --- a/samples/go/xml-import/xml_importer.go +++ b/samples/go/xml-import/xml_importer.go @@ -9,10 +9,13 @@ import ( "log" "os" "path/filepath" + "reflect" "runtime" "sort" "sync" + "github.com/stretchr/testify/assert" + "github.com/attic-labs/kingpin" "github.com/attic-labs/noms/go/config" "github.com/attic-labs/noms/go/d" @@ -21,7 +24,7 @@ import ( "github.com/attic-labs/noms/go/types" jsontonoms "github.com/attic-labs/noms/go/util/json" "github.com/attic-labs/noms/go/util/profile" - "github.com/attic-labs/noms/go/util/verbose" + "github.com/attic-labs/noms/go/util/verbose/verboseflags" "github.com/clbanning/mxj" ) @@ -50,7 +53,7 @@ func (a refIndexList) Less(i, j int) bool { return a[i].index < a[j].index } func main() { err := d.Try(func() { - verbose.RegisterVerboseFlags(kingpin.CommandLine) + verboseflags.Register(kingpin.CommandLine) profile.RegisterProfileFlags(kingpin.CommandLine) kingpin.Parse() @@ -100,7 +103,8 @@ func main() { file.Close() nomsObj := jsontonoms.NomsValueFromDecodedJSON(db, object, false) - d.Chk.IsType(expectedType, nomsObj) + d.Chk.True(assert.ObjectsAreEqual( + reflect.TypeOf(expectedType), reflect.TypeOf(nomsObj))) var r types.Ref if !*noIO {