Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

opt(builder): Use z.Allocator for building tables #1576

Merged
merged 28 commits into from
Nov 3, 2020
Merged

opt(builder): Use z.Allocator for building tables #1576

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
434928c
Basic working z.buffer
ahsanbarkati Oct 28, 2020
1ecde72
Cleanup
ahsanbarkati Oct 28, 2020
43254e6
Cleanup
ahsanbarkati Oct 28, 2020
8381b8f
Add comment
ahsanbarkati Oct 28, 2020
0c96aae
Use stream reader to make test fast. Also switch to using allocator i…
manishrjain Oct 28, 2020
abf7c87
Make allocator changes
ahsanbarkati Oct 29, 2020
c7dc592
Fix tests and cleanup
ahsanbarkati Oct 29, 2020
66d21aa
Allocate curBlock if needed
ahsanbarkati Oct 29, 2020
7f29ec5
Fix table size
ahsanbarkati Oct 29, 2020
5ae1659
Use z.Buffer for final copy.
manishrjain Oct 29, 2020
0669741
Add a TODO around logic
manishrjain Oct 29, 2020
ac241c7
Fix race conditions and cleanup
ahsanbarkati Oct 29, 2020
ad03afe
Return data back to allocator when not needed.
manishrjain Oct 30, 2020
af2c607
Reuse allocators
manishrjain Oct 30, 2020
1507408
Avoid an extra copy of the table data.
manishrjain Nov 2, 2020
233559a
Merge branch 'master' into ahsan/mmap-builder
manishrjain Nov 2, 2020
d447fee
Bring in latest ristretto
manishrjain Nov 3, 2020
9eecf5c
Create exact size file as needed and copy build data over.
manishrjain Nov 3, 2020
dbc1147
Fix TestStreamWriter6 and the ReachedCapacity function
ahsanbarkati Nov 3, 2020
f7c08ab
Fix compilation in table test
Nov 3, 2020
4ad08ee
Fix compilation in badger tests
Nov 3, 2020
a4eba92
Remove builder.offsets
Nov 3, 2020
e7120fe
Don't close builder twice
Nov 3, 2020
b6a16c1
Merge branch 'master' into ahsan/mmap-builder
manishrjain Nov 3, 2020
fe74308
All tests pass
manishrjain Nov 3, 2020
f3c54bf
No need to reverse
manishrjain Nov 3, 2020
ae6e239
Bring test.sh down to only 2 funcs.
manishrjain Nov 3, 2020
9a54599
Add TODOs
manishrjain Nov 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions db2_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package badger

import (
"bytes"
"context"
"encoding/binary"
"flag"
"fmt"
Expand Down Expand Up @@ -987,27 +988,30 @@ func TestKeyCount(t *testing.T) {
defer db.Close()
writeSorted(db, N)
require.NoError(t, db.Close())
t.Logf("Writing DONE\n")

// Read the db
db2, err := Open(DefaultOptions(dir))
y.Check(err)
defer db.Close()
lastKey := -1
count := 0
db2.View(func(txn *Txn) error {
iopt := DefaultIteratorOptions
iopt.AllVersions = true
it := txn.NewIterator(iopt)
defer it.Close()
for it.Rewind(); it.Valid(); it.Next() {
count++
i := it.Item()
key := binary.BigEndian.Uint64(i.Key())

streams := make(map[uint32]int)
stream := db2.NewStream()
stream.Send = func(list *pb.KVList) error {
count += len(list.Kv)
for _, kv := range list.Kv {
last := streams[kv.StreamId]
key := binary.BigEndian.Uint64(kv.Key)
// The following should happen as we're writing sorted data.
require.Equalf(t, lastKey+1, int(key), "Expected key: %d, Found Key: %d", lastKey+1, int(key))
lastKey = int(key)
if last > 0 {
require.Equalf(t, last+1, int(key), "Expected key: %d, Found Key: %d", lastKey+1, int(key))
}
streams[kv.StreamId] = int(key)
}
return nil
})
}
require.NoError(t, stream.Orchestrate(context.Background()))
require.Equal(t, N, uint64(count))
}
2 changes: 2 additions & 0 deletions db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/dgraph-io/badger/v2/options"
"github.com/dgraph-io/badger/v2/pb"
"github.com/dgraph-io/badger/v2/y"
"github.com/dgraph-io/ristretto/z"
)

// summary is produced when DB is closed. Currently it is used only for testing.
Expand Down Expand Up @@ -2109,6 +2110,7 @@ func TestVerifyChecksum(t *testing.T) {

func TestMain(m *testing.M) {
flag.Parse()
z.StatsPrint()
os.Exit(m.Run())
}

Expand Down
1 change: 1 addition & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
Expand Down
Loading