Merge branch 'master' into boltdb

.travis.yml

@@ -1,13 +1,14 @@
 language: go
 
 go:
-  - 1.2
   - 1.3
+  - 1.3.1
   - tip
 
 install:
   - go get github.com/badgerodon/peg
   - go get github.com/barakmich/glog
+  - go get github.com/cznic/mathutil
   - go get github.com/julienschmidt/httprouter
   - go get github.com/petar/GoLLRB/llrb
   - go get github.com/peterh/liner

graph/iterator/mock_ts_test.go

@@ -36,7 +36,7 @@ func (qs *store) ValueOf(s string) graph.Value {
 	return nil
 }
 
-func (qs *store) ApplyDeltas([]*graph.Delta) error { return nil }
+func (qs *store) ApplyDeltas([]graph.Delta) error { return nil }
 
 func (qs *store) Quad(graph.Value) quad.Quad { return quad.Quad{} }
 

graph/leveldb/iterator.go

@@ -43,9 +43,9 @@ type Iterator struct {
 	result         graph.Value
 }
 
-func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) graph.Iterator {
+func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
 	vb := value.(Token)
-	p := make([]byte, 0, 2+qs.hasherSize)
+	p := make([]byte, 0, 2+hashSize)
 	p = append(p, []byte(prefix)...)
 	p = append(p, []byte(vb[1:])...)
 
@@ -70,7 +70,6 @@ func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleS
 		it.open = false
 		it.iter.Release()
 		glog.Error("Opening LevelDB iterator couldn't seek to location ", it.nextPrefix)
-		return &iterator.Null{}
 	}
 
 	return &it
@@ -108,7 +107,7 @@ func (it *Iterator) TagResults(dst map[string]graph.Value) {
 
 func (it *Iterator) Clone() graph.Iterator {
 	out := NewIterator(it.originalPrefix, it.dir, Token(it.checkId), it.qs)
-	out.Tagger().CopyFrom(it)
+	out.tags.CopyFrom(it)
 	return out
 }
 
@@ -180,45 +179,45 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
 		case quad.Subject:
 			return 2
 		case quad.Predicate:
-			return qs.hasherSize + 2
+			return hashSize + 2
 		case quad.Object:
-			return 2*qs.hasherSize + 2
+			return 2*hashSize + 2
 		case quad.Label:
-			return 3*qs.hasherSize + 2
+			return 3*hashSize + 2
 		}
 	}
 	if bytes.Equal(prefix, []byte("po")) {
 		switch d {
 		case quad.Subject:
-			return 2*qs.hasherSize + 2
+			return 2*hashSize + 2
 		case quad.Predicate:
 			return 2
 		case quad.Object:
-			return qs.hasherSize + 2
+			return hashSize + 2
 		case quad.Label:
-			return 3*qs.hasherSize + 2
+			return hashSize + 2
 		}
 	}
 	if bytes.Equal(prefix, []byte("os")) {
 		switch d {
 		case quad.Subject:
-			return qs.hasherSize + 2
+			return hashSize + 2
 		case quad.Predicate:
-			return 2*qs.hasherSize + 2
+			return 2*hashSize + 2
 		case quad.Object:
 			return 2
 		case quad.Label:
-			return 3*qs.hasherSize + 2
+			return 3*hashSize + 2
 		}
 	}
 	if bytes.Equal(prefix, []byte("cp")) {
 		switch d {
 		case quad.Subject:
-			return 2*qs.hasherSize + 2
+			return 2*hashSize + 2
 		case quad.Predicate:
-			return qs.hasherSize + 2
+			return hashSize + 2
 		case quad.Object:
-			return 3*qs.hasherSize + 2
+			return 3*hashSize + 2
 		case quad.Label:
 			return 2
 		}

graph/leveldb/triplestore.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"hash"
+	"sync"
 
 	"github.com/barakmich/glog"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -43,24 +44,28 @@ const (
 	DefaultWriteBufferSize = 20
 )
 
+var (
+	hashPool = sync.Pool{
+		New: func() interface{} { return sha1.New() },
+	}
+	hashSize = sha1.Size
+)
+
 type Token []byte
 
 func (t Token) Key() interface{} {
 	return string(t)
 }
 
 type TripleStore struct {
-	dbOpts     *opt.Options
-	db         *leveldb.DB
-	path       string
-	open       bool
-	size       int64
-	horizon    int64
-	hasher     hash.Hash
-	hasherSize int
-	makeHasher func() hash.Hash
-	writeopts  *opt.WriteOptions
-	readopts   *opt.ReadOptions
+	dbOpts    *opt.Options
+	db        *leveldb.DB
+	path      string
+	open      bool
+	size      int64
+	horizon   int64
+	writeopts *opt.WriteOptions
+	readopts  *opt.ReadOptions
 }
 
 func createNewLevelDB(path string, _ graph.Options) error {
@@ -98,8 +103,6 @@ func newTripleStore(path string, options graph.Options) (graph.TripleStore, erro
 		write_buffer_mb = val
 	}
 	qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
-	qs.hasherSize = sha1.Size
-	qs.makeHasher = sha1.New
 	qs.writeopts = &opt.WriteOptions{
 		Sync: false,
 	}
@@ -136,30 +139,28 @@ func (qs *TripleStore) Horizon() int64 {
 	return qs.horizon
 }
 
-func (qa *TripleStore) createDeltaKeyFor(d *graph.Delta) []byte {
+func (qa *TripleStore) createDeltaKeyFor(d graph.Delta) []byte {
 	key := make([]byte, 0, 19)
 	key = append(key, 'd')
 	key = append(key, []byte(fmt.Sprintf("%018x", d.ID))...)
 	return key
 }
 
 func (qs *TripleStore) createKeyFor(d [4]quad.Direction, triple quad.Quad) []byte {
-	hasher := qs.makeHasher()
-	key := make([]byte, 0, 2+(qs.hasherSize*3))
+	key := make([]byte, 0, 2+(hashSize*3))
 	// TODO(kortschak) Remove dependence on String() method.
 	key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
-	key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
-	key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
-	key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
-	key = append(key, qs.convertStringToByteHash(triple.Get(d[3]), hasher)...)
+	key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
+	key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
+	key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
+	key = append(key, qs.convertStringToByteHash(triple.Get(d[3]))...)
 	return key
 }
 
 func (qs *TripleStore) createValueKeyFor(s string) []byte {
-	hasher := qs.makeHasher()
-	key := make([]byte, 0, 1+qs.hasherSize)
+	key := make([]byte, 0, 1+hashSize)
 	key = append(key, []byte("z")...)
-	key = append(key, qs.convertStringToByteHash(s, hasher)...)
+	key = append(key, qs.convertStringToByteHash(s)...)
 	return key
 }
 
@@ -176,7 +177,7 @@ var (
 	cps = [4]quad.Direction{quad.Label, quad.Predicate, quad.Subject, quad.Object}
 )
 
-func (qs *TripleStore) ApplyDeltas(deltas []*graph.Delta) error {
+func (qs *TripleStore) ApplyDeltas(deltas []graph.Delta) error {
 	batch := &leveldb.Batch{}
 	resizeMap := make(map[string]int64)
 	size_change := int64(0)
@@ -346,11 +347,13 @@ func (qs *TripleStore) Quad(k graph.Value) quad.Quad {
 	return triple
 }
 
-func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) []byte {
-	hasher.Reset()
-	key := make([]byte, 0, qs.hasherSize)
-	hasher.Write([]byte(s))
-	key = hasher.Sum(key)
+func (qs *TripleStore) convertStringToByteHash(s string) []byte {
+	h := hashPool.Get().(hash.Hash)
+	h.Reset()
+	defer hashPool.Put(h)
+	key := make([]byte, 0, hashSize)
+	h.Write([]byte(s))
+	key = h.Sum(key)
 	return key
 }
 
@@ -467,7 +470,7 @@ func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.
 	v := val.(Token)
 	offset := PositionOf(v[0:2], d, qs)
 	if offset != -1 {
-		return Token(append([]byte("z"), v[offset:offset+qs.hasherSize]...))
+		return Token(append([]byte("z"), v[offset:offset+hashSize]...))
 	} else {
 		return Token(qs.Quad(val).Get(d))
 	}

graph/memstore/all_iterator.go

@@ -35,17 +35,17 @@ func NewMemstoreNodesAllIterator(ts *TripleStore) *NodesAllIterator {
 }
 
 // No subiterators.
-func (nit *NodesAllIterator) SubIterators() []graph.Iterator {
+func (it *NodesAllIterator) SubIterators() []graph.Iterator {
 	return nil
 }
 
-func (nit *NodesAllIterator) Next() bool {
-	if !nit.Int64.Next() {
+func (it *NodesAllIterator) Next() bool {
+	if !it.Int64.Next() {
 		return false
 	}
-	_, ok := nit.ts.revIdMap[nit.Int64.Result().(int64)]
+	_, ok := it.ts.revIdMap[it.Int64.Result().(int64)]
 	if !ok {
-		return nit.Next()
+		return it.Next()
 	}
 	return true
 }