added support for explicit compaction

.changelog/unreleased/breaking-changes/111-compaction-support.md

@@ -0,0 +1 @@
+- Expanded db interface to support compaction ([\#111](https://github.com/cometbft/cometbft-db/pull/111))

.changelog/unreleased/features/111-compaction-support.md

@@ -0,0 +1 @@
+- Add compaction support to the databases ([\#111](https://github.com/cometbft/cometbft-db/pull/111))

backend_test.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -37,9 +39,10 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) {
 	// Default
 	dirname, err := os.MkdirTemp("", fmt.Sprintf("test_backend_%s_", backend))
 	require.Nil(t, err)
-	db, err := NewDB("testdb", backend, dirname)
+	name := fmt.Sprintf("testdb_%x", randStr(12))
+	db, err := NewDB(name, backend, dirname)
 	require.NoError(t, err)
-	defer cleanupDBDir(dirname, "testdb")
+	defer cleanupDBDir(dirname, name)
 
 	// A nonexistent key should return nil.
 	value, err := db.Get([]byte("a"))
@@ -133,6 +136,23 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) {
 	value, err = db.Get([]byte("x"))
 	require.NoError(t, err)
 	require.Equal(t, []byte{}, value)
+
+	err = db.Compact(nil, nil)
+	if strings.Contains(string(backend), "pebbledb") {
+		// In pebble the start and end will be the same so
+		// we expect an error
+		require.Error(t, err)
+	}
+
+	err = db.Set([]byte("y"), []byte{})
+	require.NoError(t, err)
+
+	err = db.Compact(nil, nil)
+	require.NoError(t, err)
+
+	if strings.Contains(string(backend), "pebbledb") {
+		time.Sleep(time.Second * 5)
+	}
 }
 
 func TestBackendsGetSetDelete(t *testing.T) {
@@ -306,6 +326,7 @@ func testDBIterator(t *testing.T, backend BackendType) {
 	// Ensure that the iterators don't panic with an empty database.
 	dir2, err := os.MkdirTemp("", "tm-db-test")
 	require.NoError(t, err)
+	name = fmt.Sprintf("test_%x", randStr(12))
 	db2, err := NewDB(name, backend, dir2)
 	require.NoError(t, err)
 	defer cleanupDBDir(dir2, name)
@@ -415,6 +436,12 @@ func testDBBatch(t *testing.T, backend BackendType) {
 	require.Error(t, batch.Delete([]byte("a")))
 	require.Error(t, batch.Write())
 	require.Error(t, batch.WriteSync())
+
+	require.NoError(t, batch.Compact(nil, nil))
+
+	if strings.Contains(string(backend), "pebbledb") {
+		time.Sleep(5 * time.Second)
+	}
 }
 
 func assertKeyValues(t *testing.T, db DB, expect map[string][]byte) {

badger_db.go

@@ -170,6 +170,11 @@ func (b *BadgerDB) Stats() map[string]string {
 	return nil
 }
 
+func (b *BadgerDB) Compact(start, end []byte) error {
+	// Explicit compaction is not currently supported in badger
+	return nil
+}
+
 func (b *BadgerDB) NewBatch() Batch {
 	wb := &badgerDBBatch{
 		db:         b.db,
@@ -225,6 +230,10 @@ func (b *badgerDBBatch) WriteSync() error {
 	return withSync(b.db, b.Write())
 }
 
+func (*badgerDBBatch) Compact(start, end []byte) error {
+	return nil
+}
+
 func (b *badgerDBBatch) Close() error {
 	select {
 	case <-b.firstFlush: // a Flush after Cancel panics too

boltdb.go

@@ -203,3 +203,9 @@ func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) {
 	}
 	return newBoltDBIterator(tx, start, end, true), nil
 }
+
+func (bdb *BoltDB) Compact(start, end []byte) error {
+	// There is no explicit CompactRange support in BoltDB, only a function that copies the
+	// entire DB from one place to another while doing deletions. Hence we do not support it.
+	return nil
+}

boltdb_batch.go

@@ -85,3 +85,7 @@ func (b *boltDBBatch) Close() error {
 	b.ops = nil
 	return nil
 }
+
+func (*boltDBBatch) Compact(start, end []byte) error {
+	return nil
+}

cleveldb.go

@@ -122,6 +122,13 @@ func (db *CLevelDB) DeleteSync(key []byte) error {
 	return nil
 }
 
+// CompactRange implements DB and compacts the given range of the DB
+func (db *CLevelDB) Compact(start, end []byte) error {
+	// CompactRange of clevelDB does not return anything
+	db.db.CompactRange(levigo.Range{Start: start, Limit: end})
+	return nil
+}
+
 // FIXME This should not be exposed
 func (db *CLevelDB) DB() *levigo.DB {
 	return db.db

cleveldb_batch.go

@@ -80,3 +80,8 @@ func (b *cLevelDBBatch) Close() error {
 	}
 	return nil
 }
+
+// Compact compacts given range of DB
+func (b *cLevelDBBatch) Compact(start, end []byte) error {
+	return b.db.Compact(start, end)
+}

go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/kr/pretty v0.2.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -40,7 +41,6 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/sys v0.14.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect

goleveldb.go

@@ -33,6 +33,7 @@ func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB,
 	if err != nil {
 		return nil, err
 	}
+
 	database := &GoLevelDB{
 		db: db,
 	}
@@ -71,7 +72,8 @@ func (db *GoLevelDB) Set(key []byte, value []byte) error {
 	if value == nil {
 		return errValueNil
 	}
-	if err := db.db.Put(key, value, nil); err != nil {
+	err := db.db.Put(key, value, nil)
+	if err != nil {
 		return err
 	}
 	return nil
@@ -85,7 +87,9 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) error {
 	if value == nil {
 		return errValueNil
 	}
-	if err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}); err != nil {
+
+	err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
+	if err != nil {
 		return err
 	}
 	return nil
@@ -96,7 +100,9 @@ func (db *GoLevelDB) Delete(key []byte) error {
 	if len(key) == 0 {
 		return errKeyEmpty
 	}
-	if err := db.db.Delete(key, nil); err != nil {
+
+	err := db.db.Delete(key, nil)
+	if err != nil {
 		return err
 	}
 	return nil
@@ -188,3 +194,8 @@ func (db *GoLevelDB) ReverseIterator(start, end []byte) (Iterator, error) {
 	itr := db.db.NewIterator(&util.Range{Start: start, Limit: end}, nil)
 	return newGoLevelDBIterator(itr, start, end, true), nil
 }
+
+// Compact range
+func (db *GoLevelDB) Compact(start, end []byte) error {
+	return db.db.CompactRange(util.Range{Start: start, Limit: end})
+}

goleveldb_batch.go

@@ -3,6 +3,7 @@ package db
 import (
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 type goLevelDBBatch struct {
@@ -60,6 +61,7 @@ func (b *goLevelDBBatch) write(sync bool) error {
 	if b.batch == nil {
 		return errBatchClosed
 	}
+
 	err := b.db.db.Write(b.batch, &opt.WriteOptions{Sync: sync})
 	if err != nil {
 		return err
@@ -76,3 +78,7 @@ func (b *goLevelDBBatch) Close() error {
 	}
 	return nil
 }
+
+func (b *goLevelDBBatch) Compact(start, end []byte) error {
+	return b.db.db.CompactRange(util.Range{Start: start, Limit: end})
+}

memdb.go

@@ -208,3 +208,8 @@ func (db *MemDB) ReverseIteratorNoMtx(start, end []byte) (Iterator, error) {
 	}
 	return newMemDBIteratorMtxChoice(db, start, end, true, false), nil
 }
+
+func (*MemDB) Compact(start, end []byte) error {
+	// No Compaction is supported for memDB and there is no point in supporting compaction for a memory DB
+	return nil
+}

memdb_batch.go

@@ -92,3 +92,7 @@ func (b *memDBBatch) Close() error {
 	b.ops = nil
 	return nil
 }
+
+func (*memDBBatch) Compact(start, end []byte) error {
+	return nil
+}

pebble.go

@@ -136,6 +136,29 @@
 	return db.db
 }
 
+func (db *PebbleDB) Compact(start, end []byte) error {
+	// Currently nil,nil is an invalid range in Pebble.
+	// This was taken from https://github.com/cockroachdb/pebble/issues/1474
+	// In case the start and end keys are the same
+	// pebbleDB will throw an error that it cannot compact.
+	if start != nil && end != nil {
+		return db.db.Compact(start, end, true)
+	}
+
+	iter := db.db.NewIter(nil)
+
+	if start == nil && iter.First() {
+		start = append(start, iter.Key()...)
+	}
+	if end == nil && iter.Last() {
+		end = append(end, iter.Key()...)
+	}
+	if err := iter.Close(); err != nil {
+		return err
+	}
+	return db.db.Compact(start, end, true)
+}
+
 // Close implements DB.
 func (db PebbleDB) Close() error {
 	db.db.Close()
@@ -285,6 +308,12 @@
 	return nil
 }
 
+// Compact specified range
+// nil, nil will run compaction over the entire database
+func (b *pebbleDBBatch) Compact(start, end []byte) error {
+	return b.db.Compact(start, end)
+}
+
 type pebbleDBIterator struct {
 	source     *pebble.Iterator
 	start, end []byte

prefixdb.go

@@ -202,3 +202,7 @@ func (pdb *PrefixDB) Stats() map[string]string {
 func (pdb *PrefixDB) prefixed(key []byte) []byte {
 	return append(cp(pdb.prefix), key...)
 }
+
+func (pdb *PrefixDB) Compact(start, end []byte) error {
+	return pdb.db.Compact(start, end)
+}

prefixdb_batch.go

@@ -49,3 +49,8 @@ func (pb prefixDBBatch) WriteSync() error {
 func (pb prefixDBBatch) Close() error {
 	return pb.source.Close()
 }
+
+func (prefixDBBatch) Compact(start, end []byte) error {
+	// Not implemented
+	return nil
+}

rocksdb.go

@@ -202,3 +202,8 @@ func (db *RocksDB) ReverseIterator(start, end []byte) (Iterator, error) {
 	itr := db.db.NewIterator(db.ro)
 	return newRocksDBIterator(itr, start, end, true), nil
 }
+
+func (db *RocksDB) Compact(start, end []byte) error {
+	db.db.CompactRange(grocksdb.Range{Start: start, Limit: end})
+	return nil
+}

rocksdb_batch.go

@@ -81,3 +81,8 @@ func (b *rocksDBBatch) Close() error {
 	}
 	return nil
 }
+
+// Compact compacts a given range of keys in the DB
+func (b *rocksDBBatch) Compact(start, end []byte) error {
+	return b.db.Compact(start, end)
+}

types.go

@@ -68,6 +68,9 @@ type DB interface {
 
 	// Stats returns a map of property values for all keys and the size of the cache.
 	Stats() map[string]string
+
+	// Compact explicitly
+	Compact(start, end []byte) error
 }
 
 // Batch represents a group of writes. They may or may not be written atomically depending on the
@@ -94,6 +97,9 @@ type Batch interface {
 
 	// Close closes the batch. It is idempotent, but calls to other methods afterwards will error.
 	Close() error
+
+	// Compact explicitly
+	Compact(start, end []byte) error
 }
 
 // Iterator represents an iterator over a domain of keys. Callers must call Close when done.