From 6b1bd3301dd4ee5bf84b372de91f2b7e930e4170 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Wed, 3 Jan 2024 16:02:15 +0100 Subject: [PATCH 01/19] Added compaction to bolt and badger --- backend_test.go | 6 ++- badger_db.go | 5 +++ boltdb.go | 6 +++ cleveldb.go | 1 + go.mod | 7 ++++ go.sum | 26 ++++++++++-- goleveldb.go | 94 ++++++++++++++++++++++++++++++++++++++++++-- goleveldb_batch.go | 9 +++++ memdb.go | 5 +++ prefixdb.go | 4 ++ remotedb/remotedb.go | 4 ++ rocksdb.go | 5 +++ types.go | 3 ++ 13 files changed, 167 insertions(+), 8 deletions(-) diff --git a/backend_test.go b/backend_test.go index 0f19241..9eef3b2 100644 --- a/backend_test.go +++ b/backend_test.go @@ -36,9 +36,10 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { // Default dirname, err := os.MkdirTemp("", fmt.Sprintf("test_backend_%s_", backend)) require.Nil(t, err) - db, err := NewDB("testdb", backend, dirname) + name := fmt.Sprintf("testdb_%x", randStr(12)) + db, err := NewDB(name, backend, dirname) require.NoError(t, err) - defer cleanupDBDir(dirname, "testdb") + defer cleanupDBDir(dirname, name) // A nonexistent key should return nil. value, err := db.Get([]byte("a")) @@ -303,6 +304,7 @@ func testDBIterator(t *testing.T, backend BackendType) { // Ensure that the iterators don't panic with an empty database. dir2, err := os.MkdirTemp("", "tm-db-test") require.NoError(t, err) + name = fmt.Sprintf("test_%x", randStr(12)) db2, err := NewDB(name, backend, dir2) require.NoError(t, err) defer cleanupDBDir(dir2, name) diff --git a/badger_db.go b/badger_db.go index ecdfca1..f0e27cc 100644 --- a/badger_db.go +++ b/badger_db.go @@ -170,6 +170,11 @@ func (b *BadgerDB) Stats() map[string]string { return nil } +func (b *BadgerDB) Compact(start, end []byte) error { + // Explicit compaction is not currently supported in badger + return nil +} + func (b *BadgerDB) NewBatch() Batch { wb := &badgerDBBatch{ db: b.db, diff --git a/boltdb.go b/boltdb.go index ffebbca..4738fca 100644 --- a/boltdb.go +++ b/boltdb.go @@ -205,3 +205,9 @@ func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) { } return newBoltDBIterator(tx, start, end, true), nil } + +func (bdb *BoltDB) Compact(start, end []byte) error { + // There is no explicit CompactRange support in BoltDB, only a function that copies the + // entire DB from one place to another while doing deletions. Hence we do not support it. + return nil +} diff --git a/cleveldb.go b/cleveldb.go index 7896730..8240f8f 100644 --- a/cleveldb.go +++ b/cleveldb.go @@ -48,6 +48,7 @@ func NewCLevelDB(name string, dir string) (*CLevelDB, error) { wo: wo, woSync: woSync, } + return database, nil } diff --git a/go.mod b/go.mod index f9e709a..0eeec2a 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/btree v1.1.2 github.com/jmhodges/levigo v1.0.0 github.com/linxGnu/grocksdb v1.8.10 + github.com/prometheus/client_golang v1.17.0 github.com/stretchr/testify v1.8.4 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca go.etcd.io/bbolt v1.3.8 @@ -15,7 +16,9 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect @@ -23,8 +26,12 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/klauspost/compress v1.12.3 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/pkg/errors v0.8.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index eea5c31..c989ef5 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -59,14 +63,17 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/linxGnu/grocksdb v1.8.10 h1:6FAhBThErRfJaevGOZISYvkG7RD4gfzeq452X4r8pes= github.com/linxGnu/grocksdb v1.8.10/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= @@ -83,6 +90,16 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -122,6 +139,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -168,14 +186,16 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/goleveldb.go b/goleveldb.go index fd1bffd..9628d71 100644 --- a/goleveldb.go +++ b/goleveldb.go @@ -3,11 +3,14 @@ package db import ( "fmt" "path/filepath" + "time" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" + + "github.com/prometheus/client_golang/prometheus" ) func init() { @@ -17,8 +20,18 @@ func init() { registerDBCreator(GoLevelDBBackend, dbCreator, false) } +const PROMETHEUS_NAMESPACE = "cometbft_db" + type GoLevelDB struct { db *leveldb.DB + // All durations are reported in milliseconds. + getDuration prometheus.Gauge + setDuration prometheus.Gauge + setSyncDuration prometheus.Gauge + deleteDuration prometheus.Gauge + deleteSyncDuration prometheus.Gauge + batchDuration prometheus.Gauge + batchSyncDuration prometheus.Gauge } var _ DB = (*GoLevelDB)(nil) @@ -33,9 +46,17 @@ func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, if err != nil { return nil, err } + + // Create a new levelDBCollector + collector := newLevelDBCollector(db, name) + // Register the collector with Prometheus + prometheus.MustRegister(collector) + database := &GoLevelDB{ db: db, } + + database.createPrometheusMetrics(name) return database, nil } @@ -44,7 +65,9 @@ func (db *GoLevelDB) Get(key []byte) ([]byte, error) { if len(key) == 0 { return nil, errKeyEmpty } + start := time.Now() res, err := db.db.Get(key, nil) + db.getDuration.Set(float64(time.Since(start).Milliseconds())) if err != nil { if err == errors.ErrNotFound { return nil, nil @@ -71,7 +94,10 @@ func (db *GoLevelDB) Set(key []byte, value []byte) error { if value == nil { return errValueNil } - if err := db.db.Put(key, value, nil); err != nil { + start := time.Now() + err := db.db.Put(key, value, nil) + db.setDuration.Set(float64(time.Since(start).Milliseconds())) + if err != nil { return err } return nil @@ -85,7 +111,10 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) error { if value == nil { return errValueNil } - if err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}); err != nil { + start := time.Now() + err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) + db.setSyncDuration.Set(float64(time.Since(start).Milliseconds())) + if err != nil { return err } return nil @@ -96,7 +125,10 @@ func (db *GoLevelDB) Delete(key []byte) error { if len(key) == 0 { return errKeyEmpty } - if err := db.db.Delete(key, nil); err != nil { + start := time.Now() + err := db.db.Delete(key, nil) + db.deleteDuration.Set(float64(time.Since(start).Milliseconds())) + if err != nil { return err } return nil @@ -188,3 +220,59 @@ func (db *GoLevelDB) ReverseIterator(start, end []byte) (Iterator, error) { itr := db.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) return newGoLevelDBIterator(itr, start, end, true), nil } + +func (db *GoLevelDB) Compact(start, end []byte) error { + fmt.Println("Compacting") + return db.db.CompactRange(util.Range{Start: start, Limit: end}) +} +func (db *GoLevelDB) createPrometheusMetrics(dbName string) { + db.getDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: PROMETHEUS_NAMESPACE, + Subsystem: dbName, + Name: "get_duration_ms", + Help: "The duration of the Get() operation in ms.", + }) + prometheus.MustRegister(db.getDuration) + db.setDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: PROMETHEUS_NAMESPACE, + Subsystem: dbName, + Name: "set_duration_ms", + Help: "The duration of the Set() operation in ms.", + }) + prometheus.MustRegister(db.setDuration) + db.setSyncDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: PROMETHEUS_NAMESPACE, + Subsystem: dbName, + Name: "set_sync_duration_ms", + Help: "The duration of the SetSync() operation in ms.", + }) + prometheus.MustRegister(db.setSyncDuration) + db.deleteDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: PROMETHEUS_NAMESPACE, + Subsystem: dbName, + Name: "delete_duration_ms", + Help: "The duration of the Delete() operation in ms.", + }) + prometheus.MustRegister(db.deleteDuration) + db.deleteSyncDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: PROMETHEUS_NAMESPACE, + Subsystem: dbName, + Name: "delete_sync_duration_ms", + Help: "The duration of the DeleteSync() operation in ms.", + }) + prometheus.MustRegister(db.deleteSyncDuration) + db.batchDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: PROMETHEUS_NAMESPACE, + Subsystem: dbName, + Name: "batch_duration_ms", + Help: "The duration of the batch#write operation in ms.", + }) + prometheus.MustRegister(db.batchDuration) + db.batchSyncDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: PROMETHEUS_NAMESPACE, + Subsystem: dbName, + Name: "batch_sync_duration_ms", + Help: "The duration of the batch#write(sync) operation in ms.", + }) + prometheus.MustRegister(db.batchSyncDuration) +} diff --git a/goleveldb_batch.go b/goleveldb_batch.go index 4c1c6a2..2ed9b61 100644 --- a/goleveldb_batch.go +++ b/goleveldb_batch.go @@ -3,6 +3,7 @@ package db import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" + "time" ) type goLevelDBBatch struct { @@ -60,6 +61,14 @@ func (b *goLevelDBBatch) write(sync bool) error { if b.batch == nil { return errBatchClosed } + start := time.Now() + defer func() { + if sync { + b.db.batchSyncDuration.Set(float64(time.Since(start).Milliseconds())) + } else { + b.db.batchDuration.Set(float64(time.Since(start).Milliseconds())) + } + }() err := b.db.db.Write(b.batch, &opt.WriteOptions{Sync: sync}) if err != nil { return err diff --git a/memdb.go b/memdb.go index 2bfa0d1..53b238b 100644 --- a/memdb.go +++ b/memdb.go @@ -205,3 +205,8 @@ func (db *MemDB) ReverseIteratorNoMtx(start, end []byte) (Iterator, error) { } return newMemDBIteratorMtxChoice(db, start, end, true, false), nil } + +func (db *MemDB) Compact(start, end []byte) error { + // No Compaction is supported for memDB and there is no point in supporting compaction for a memory DB + return nil +} diff --git a/prefixdb.go b/prefixdb.go index 0b2d2a1..30e29a5 100644 --- a/prefixdb.go +++ b/prefixdb.go @@ -202,3 +202,7 @@ func (pdb *PrefixDB) Stats() map[string]string { func (pdb *PrefixDB) prefixed(key []byte) []byte { return append(cp(pdb.prefix), key...) } + +func (pdb *PrefixDB) Compact(start, end []byte) error { + return pdb.db.Compact(start, end) +} diff --git a/remotedb/remotedb.go b/remotedb/remotedb.go index 7b1c8da..27a5e88 100644 --- a/remotedb/remotedb.go +++ b/remotedb/remotedb.go @@ -121,3 +121,7 @@ func (rd *RemoteDB) Iterator(start, end []byte) (db.Iterator, error) { } return makeIterator(dic), nil } + +func (rd *RemoteDB) Compact(start, end []byte) error { + return nil +} diff --git a/rocksdb.go b/rocksdb.go index 945491c..abc1185 100644 --- a/rocksdb.go +++ b/rocksdb.go @@ -202,3 +202,8 @@ func (db *RocksDB) ReverseIterator(start, end []byte) (Iterator, error) { itr := db.db.NewIterator(db.ro) return newRocksDBIterator(itr, start, end, true), nil } + +func (db *RocksDB) Compact(start, end []byte) error { + db.db.CompactRange(grocksdb.Range{Start: start, Limit: end}) + return nil +} diff --git a/types.go b/types.go index 131174d..74da21e 100644 --- a/types.go +++ b/types.go @@ -68,6 +68,9 @@ type DB interface { // Stats returns a map of property values for all keys and the size of the cache. Stats() map[string]string + + // Compact explicitly + Compact(start, end []byte) error } // Batch represents a group of writes. They may or may not be written atomically depending on the From fc38cdf873e545157f9dc250978e67165f6608f4 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Mon, 29 Jan 2024 14:38:35 +0100 Subject: [PATCH 02/19] Added compaction to pebble and removed stats from goleveldb --- go.mod | 2 +- goleveldb.go | 87 ++++------------------------------------------ goleveldb_batch.go | 16 ++++----- memdb.go | 2 +- pebble.go | 10 ++++++ 5 files changed, 26 insertions(+), 91 deletions(-) diff --git a/go.mod b/go.mod index b28789b..e68a286 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -40,7 +41,6 @@ require ( github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect - github.com/google/go-cmp v0.6.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/goleveldb.go b/goleveldb.go index df2a7a9..06533b3 100644 --- a/goleveldb.go +++ b/goleveldb.go @@ -3,14 +3,11 @@ package db import ( "fmt" "path/filepath" - "time" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" - - "github.com/prometheus/client_golang/prometheus" ) func init() { @@ -20,18 +17,8 @@ func init() { registerDBCreator(GoLevelDBBackend, dbCreator) } -const PROMETHEUS_NAMESPACE = "cometbft_db" - type GoLevelDB struct { db *leveldb.DB - // All durations are reported in milliseconds. - getDuration prometheus.Gauge - setDuration prometheus.Gauge - setSyncDuration prometheus.Gauge - deleteDuration prometheus.Gauge - deleteSyncDuration prometheus.Gauge - batchDuration prometheus.Gauge - batchSyncDuration prometheus.Gauge } var _ DB = (*GoLevelDB)(nil) @@ -47,16 +34,9 @@ func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, return nil, err } - // Create a new levelDBCollector - collector := newLevelDBCollector(db, name) - // Register the collector with Prometheus - prometheus.MustRegister(collector) - database := &GoLevelDB{ db: db, } - - database.createPrometheusMetrics(name) return database, nil } @@ -65,9 +45,8 @@ func (db *GoLevelDB) Get(key []byte) ([]byte, error) { if len(key) == 0 { return nil, errKeyEmpty } - start := time.Now() res, err := db.db.Get(key, nil) - db.getDuration.Set(float64(time.Since(start).Milliseconds())) + if err != nil { if err == errors.ErrNotFound { return nil, nil @@ -94,9 +73,8 @@ func (db *GoLevelDB) Set(key []byte, value []byte) error { if value == nil { return errValueNil } - start := time.Now() err := db.db.Put(key, value, nil) - db.setDuration.Set(float64(time.Since(start).Milliseconds())) + if err != nil { return err } @@ -111,9 +89,9 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) error { if value == nil { return errValueNil } - start := time.Now() + err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) - db.setSyncDuration.Set(float64(time.Since(start).Milliseconds())) + if err != nil { return err } @@ -125,9 +103,9 @@ func (db *GoLevelDB) Delete(key []byte) error { if len(key) == 0 { return errKeyEmpty } - start := time.Now() + err := db.db.Delete(key, nil) - db.deleteDuration.Set(float64(time.Since(start).Milliseconds())) + if err != nil { return err } @@ -221,58 +199,7 @@ func (db *GoLevelDB) ReverseIterator(start, end []byte) (Iterator, error) { return newGoLevelDBIterator(itr, start, end, true), nil } +// Compact range func (db *GoLevelDB) Compact(start, end []byte) error { - fmt.Println("Compacting") return db.db.CompactRange(util.Range{Start: start, Limit: end}) } -func (db *GoLevelDB) createPrometheusMetrics(dbName string) { - db.getDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: PROMETHEUS_NAMESPACE, - Subsystem: dbName, - Name: "get_duration_ms", - Help: "The duration of the Get() operation in ms.", - }) - prometheus.MustRegister(db.getDuration) - db.setDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: PROMETHEUS_NAMESPACE, - Subsystem: dbName, - Name: "set_duration_ms", - Help: "The duration of the Set() operation in ms.", - }) - prometheus.MustRegister(db.setDuration) - db.setSyncDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: PROMETHEUS_NAMESPACE, - Subsystem: dbName, - Name: "set_sync_duration_ms", - Help: "The duration of the SetSync() operation in ms.", - }) - prometheus.MustRegister(db.setSyncDuration) - db.deleteDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: PROMETHEUS_NAMESPACE, - Subsystem: dbName, - Name: "delete_duration_ms", - Help: "The duration of the Delete() operation in ms.", - }) - prometheus.MustRegister(db.deleteDuration) - db.deleteSyncDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: PROMETHEUS_NAMESPACE, - Subsystem: dbName, - Name: "delete_sync_duration_ms", - Help: "The duration of the DeleteSync() operation in ms.", - }) - prometheus.MustRegister(db.deleteSyncDuration) - db.batchDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: PROMETHEUS_NAMESPACE, - Subsystem: dbName, - Name: "batch_duration_ms", - Help: "The duration of the batch#write operation in ms.", - }) - prometheus.MustRegister(db.batchDuration) - db.batchSyncDuration = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: PROMETHEUS_NAMESPACE, - Subsystem: dbName, - Name: "batch_sync_duration_ms", - Help: "The duration of the batch#write(sync) operation in ms.", - }) - prometheus.MustRegister(db.batchSyncDuration) -} diff --git a/goleveldb_batch.go b/goleveldb_batch.go index 2ed9b61..ec62a12 100644 --- a/goleveldb_batch.go +++ b/goleveldb_batch.go @@ -3,7 +3,7 @@ package db import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" - "time" + "github.com/syndtr/goleveldb/leveldb/util" ) type goLevelDBBatch struct { @@ -61,14 +61,7 @@ func (b *goLevelDBBatch) write(sync bool) error { if b.batch == nil { return errBatchClosed } - start := time.Now() - defer func() { - if sync { - b.db.batchSyncDuration.Set(float64(time.Since(start).Milliseconds())) - } else { - b.db.batchDuration.Set(float64(time.Since(start).Milliseconds())) - } - }() + err := b.db.db.Write(b.batch, &opt.WriteOptions{Sync: sync}) if err != nil { return err @@ -85,3 +78,8 @@ func (b *goLevelDBBatch) Close() error { } return nil } + +func (b *goLevelDBBatch) Compact(start, end []byte) error { + return b.db.db.CompactRange(util.Range{Start: start, Limit: end}) + +} diff --git a/memdb.go b/memdb.go index 04a3b80..b915504 100644 --- a/memdb.go +++ b/memdb.go @@ -209,7 +209,7 @@ func (db *MemDB) ReverseIteratorNoMtx(start, end []byte) (Iterator, error) { return newMemDBIteratorMtxChoice(db, start, end, true, false), nil } -func (db *MemDB) Compact(start, end []byte) error { +func (*MemDB) Compact(start, end []byte) error { // No Compaction is supported for memDB and there is no point in supporting compaction for a memory DB return nil } diff --git a/pebble.go b/pebble.go index 08ed5aa..b13e456 100644 --- a/pebble.go +++ b/pebble.go @@ -136,6 +136,10 @@ func (db *PebbleDB) DB() *pebble.DB { return db.db } +func (db *PebbleDB) Compact(start, end []byte) error { + return db.db.Compact(start, end, true) +} + // Close implements DB. func (db PebbleDB) Close() error { db.db.Close() @@ -280,6 +284,12 @@ func (b *pebbleDBBatch) Close() error { return nil } +// Compact specified range +// nil, nil will run compaction over the entire database +func (b *pebbleDBBatch) Compat(start, end []byte) error { + return b.db.Compact(start, end) +} + type pebbleDBIterator struct { source *pebble.Iterator start, end []byte From 89480b49bc434856559dad1562e1e5fe467f57cb Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Mon, 29 Jan 2024 14:48:09 +0100 Subject: [PATCH 03/19] Fixed lint --- goleveldb.go | 4 ---- goleveldb_batch.go | 1 - 2 files changed, 5 deletions(-) diff --git a/goleveldb.go b/goleveldb.go index 06533b3..e54f91b 100644 --- a/goleveldb.go +++ b/goleveldb.go @@ -46,7 +46,6 @@ func (db *GoLevelDB) Get(key []byte) ([]byte, error) { return nil, errKeyEmpty } res, err := db.db.Get(key, nil) - if err != nil { if err == errors.ErrNotFound { return nil, nil @@ -74,7 +73,6 @@ func (db *GoLevelDB) Set(key []byte, value []byte) error { return errValueNil } err := db.db.Put(key, value, nil) - if err != nil { return err } @@ -91,7 +89,6 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) error { } err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) - if err != nil { return err } @@ -105,7 +102,6 @@ func (db *GoLevelDB) Delete(key []byte) error { } err := db.db.Delete(key, nil) - if err != nil { return err } diff --git a/goleveldb_batch.go b/goleveldb_batch.go index ec62a12..bbe69cd 100644 --- a/goleveldb_batch.go +++ b/goleveldb_batch.go @@ -81,5 +81,4 @@ func (b *goLevelDBBatch) Close() error { func (b *goLevelDBBatch) Compact(start, end []byte) error { return b.db.db.CompactRange(util.Range{Start: start, Limit: end}) - } From 170fc4b05a56fcf5f22282b40c5923514f77ae26 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Mon, 29 Jan 2024 15:38:26 +0100 Subject: [PATCH 04/19] Added compact function to clevelDB --- cleveldb.go | 8 +++++++- cleveldb_batch.go | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cleveldb.go b/cleveldb.go index 8a786de..3fabe9f 100644 --- a/cleveldb.go +++ b/cleveldb.go @@ -48,7 +48,6 @@ func NewCLevelDB(name string, dir string) (*CLevelDB, error) { wo: wo, woSync: woSync, } - return database, nil } @@ -123,6 +122,13 @@ func (db *CLevelDB) DeleteSync(key []byte) error { return nil } +// CompactRange implements DB and compacts the given range of the DB +func (db *CLevelDB) Compact(start, end []byte) error { + // CompactRange of clevelDB does not return anything + db.db.CompactRange(levigo.Range{Start: start, Limit: end}) + return nil +} + // FIXME This should not be exposed func (db *CLevelDB) DB() *levigo.DB { return db.db diff --git a/cleveldb_batch.go b/cleveldb_batch.go index b77bd52..843fac5 100644 --- a/cleveldb_batch.go +++ b/cleveldb_batch.go @@ -80,3 +80,8 @@ func (b *cLevelDBBatch) Close() error { } return nil } + +// Compact compacts given range of DB +func (b *cLevelDBBatch) Compact(start, end []byte) error { + return b.db.Compact(start, end) +} From 9adfbd146403fd6e5fe659778d775acfe7b2429d Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Mon, 29 Jan 2024 16:08:11 +0100 Subject: [PATCH 05/19] Added compact function to batch DBs --- boltdb_batch.go | 4 ++++ memdb_batch.go | 4 ++++ prefixdb_batch.go | 5 +++++ rocksdb_batch.go | 5 +++++ 4 files changed, 18 insertions(+) diff --git a/boltdb_batch.go b/boltdb_batch.go index cd22c67..88a38e5 100644 --- a/boltdb_batch.go +++ b/boltdb_batch.go @@ -85,3 +85,7 @@ func (b *boltDBBatch) Close() error { b.ops = nil return nil } + +func (b *boltDBBatch) Compact(start, end []byte) error { + return nil +} diff --git a/memdb_batch.go b/memdb_batch.go index 6ff30fe..9829efb 100644 --- a/memdb_batch.go +++ b/memdb_batch.go @@ -92,3 +92,7 @@ func (b *memDBBatch) Close() error { b.ops = nil return nil } + +func (b *memDBBatch) Compact(start, end []byte) error { + return nil +} diff --git a/prefixdb_batch.go b/prefixdb_batch.go index 1a8005c..9806773 100644 --- a/prefixdb_batch.go +++ b/prefixdb_batch.go @@ -49,3 +49,8 @@ func (pb prefixDBBatch) WriteSync() error { func (pb prefixDBBatch) Close() error { return pb.source.Close() } + +func (pb prefixDBBatch) Compact(start, end []byte) error { + // Not implemented + return nil +} diff --git a/rocksdb_batch.go b/rocksdb_batch.go index 6ebc8da..6a55ada 100644 --- a/rocksdb_batch.go +++ b/rocksdb_batch.go @@ -81,3 +81,8 @@ func (b *rocksDBBatch) Close() error { } return nil } + +// Compact compacts a given range of keys in the DB +func (b *rocksDBBatch) Compact(start, end []byte) error { + return b.db.Compact(start, end) +} From b5497157c701573d983d8b01b5a13067842efe8a Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Mon, 29 Jan 2024 16:17:45 +0100 Subject: [PATCH 06/19] Removed unused identifiers --- boltdb_batch.go | 2 +- memdb_batch.go | 2 +- prefixdb_batch.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/boltdb_batch.go b/boltdb_batch.go index 88a38e5..40750c1 100644 --- a/boltdb_batch.go +++ b/boltdb_batch.go @@ -86,6 +86,6 @@ func (b *boltDBBatch) Close() error { return nil } -func (b *boltDBBatch) Compact(start, end []byte) error { +func (*boltDBBatch) Compact(start, end []byte) error { return nil } diff --git a/memdb_batch.go b/memdb_batch.go index 9829efb..523dc05 100644 --- a/memdb_batch.go +++ b/memdb_batch.go @@ -93,6 +93,6 @@ func (b *memDBBatch) Close() error { return nil } -func (b *memDBBatch) Compact(start, end []byte) error { +func (*memDBBatch) Compact(start, end []byte) error { return nil } diff --git a/prefixdb_batch.go b/prefixdb_batch.go index 9806773..782c295 100644 --- a/prefixdb_batch.go +++ b/prefixdb_batch.go @@ -50,7 +50,7 @@ func (pb prefixDBBatch) Close() error { return pb.source.Close() } -func (pb prefixDBBatch) Compact(start, end []byte) error { +func (prefixDBBatch) Compact(start, end []byte) error { // Not implemented return nil } From ebd78730e391e3fa36b67b2a8e1aed58a9ba2d7e Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Mon, 29 Jan 2024 16:39:22 +0100 Subject: [PATCH 07/19] Added call to compact from tests and expanded the batch interface --- backend_test.go | 4 ++++ types.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/backend_test.go b/backend_test.go index 23a5a87..670716c 100644 --- a/backend_test.go +++ b/backend_test.go @@ -134,6 +134,8 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { value, err = db.Get([]byte("x")) require.NoError(t, err) require.Equal(t, []byte{}, value) + + require.NoError(t, db.Compact(nil, nil)) } func TestBackendsGetSetDelete(t *testing.T) { @@ -417,6 +419,8 @@ func testDBBatch(t *testing.T, backend BackendType) { require.Error(t, batch.Delete([]byte("a"))) require.Error(t, batch.Write()) require.Error(t, batch.WriteSync()) + + require.NoError(t, batch.Compact(nil, nil)) } func assertKeyValues(t *testing.T, db DB, expect map[string][]byte) { diff --git a/types.go b/types.go index 74da21e..a5f6bf1 100644 --- a/types.go +++ b/types.go @@ -97,6 +97,9 @@ type Batch interface { // Close closes the batch. It is idempotent, but calls to other methods afterwards will error. Close() error + + // Compact explicitly + Compact(start, end []byte) error } // Iterator represents an iterator over a domain of keys. Callers must call Close when done. From a8118ced817e3f7fe31ddeff3dd7e65d8dafbd5d Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 30 Jan 2024 11:20:56 +0100 Subject: [PATCH 08/19] Added compacton to test --- backend_test.go | 3 ++- badger_db.go | 4 ++++ pebble.go | 19 +++++++++++++++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/backend_test.go b/backend_test.go index 670716c..6e3f119 100644 --- a/backend_test.go +++ b/backend_test.go @@ -135,7 +135,8 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { require.NoError(t, err) require.Equal(t, []byte{}, value) - require.NoError(t, db.Compact(nil, nil)) + err = db.Set([]byte("y"), []byte{}) + require.NoError(t, err) } func TestBackendsGetSetDelete(t *testing.T) { diff --git a/badger_db.go b/badger_db.go index d66f05c..ba8d53d 100644 --- a/badger_db.go +++ b/badger_db.go @@ -230,6 +230,10 @@ func (b *badgerDBBatch) WriteSync() error { return withSync(b.db, b.Write()) } +func (*badgerDBBatch) Compact(start, end []byte) error { + return nil +} + func (b *badgerDBBatch) Close() error { select { case <-b.firstFlush: // a Flush after Cancel panics too diff --git a/pebble.go b/pebble.go index 025c0f0..0ca3766 100644 --- a/pebble.go +++ b/pebble.go @@ -137,7 +137,22 @@ func (db *PebbleDB) DB() *pebble.DB { } func (db *PebbleDB) Compact(start, end []byte) error { - return db.db.Compact(start, end, true) + // Currently nil,nil is an invalid range in Pebble. + // This was taken from https://github.com/cockroachdb/pebble/issues/1474 + + iter := db.db.NewIter(nil) + var first, last []byte + + if iter.First() { + first = append(first, iter.Key()...) + } + if iter.Last() { + last = append(last, iter.Key()...) + } + if err := iter.Close(); err != nil { + return err + } + return db.db.Compact(first, last, true) } // Close implements DB. @@ -291,7 +306,7 @@ func (b *pebbleDBBatch) Close() error { // Compact specified range // nil, nil will run compaction over the entire database -func (b *pebbleDBBatch) Compat(start, end []byte) error { +func (b *pebbleDBBatch) Compact(start, end []byte) error { return b.db.Compact(start, end) } From 7292b789635386abcfc8311ce308bc85f53c9b2a Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 30 Jan 2024 11:35:42 +0100 Subject: [PATCH 09/19] Added compacton to test-fix --- backend_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend_test.go b/backend_test.go index 6e3f119..00a39fe 100644 --- a/backend_test.go +++ b/backend_test.go @@ -137,6 +137,9 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { err = db.Set([]byte("y"), []byte{}) require.NoError(t, err) + + err = db.Compact(nil, nil) + require.NoError(t, err) } func TestBackendsGetSetDelete(t *testing.T) { From e01569d95d23bbf890d04aadd57e6678552e6345 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 30 Jan 2024 14:09:58 +0100 Subject: [PATCH 10/19] Fixed tests for pebbleDB --- backend_test.go | 17 +++++++++++++++++ pebble.go | 16 ++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/backend_test.go b/backend_test.go index 00a39fe..095a0bf 100644 --- a/backend_test.go +++ b/backend_test.go @@ -4,10 +4,12 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "time" ) // Register a test backend for PrefixDB as well, with some unrelated junk data @@ -135,11 +137,22 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { require.NoError(t, err) require.Equal(t, []byte{}, value) + err = db.Compact(nil, nil) + if strings.Contains(string(backend), "pebbledb") { + // In pebble the strat and end will be the same so + // we expect an error + require.Error(t, err) + } + err = db.Set([]byte("y"), []byte{}) require.NoError(t, err) err = db.Compact(nil, nil) require.NoError(t, err) + + if strings.Contains(string(backend), "pebbledb") { + time.Sleep(time.Second * 5) + } } func TestBackendsGetSetDelete(t *testing.T) { @@ -425,6 +438,10 @@ func testDBBatch(t *testing.T, backend BackendType) { require.Error(t, batch.WriteSync()) require.NoError(t, batch.Compact(nil, nil)) + + if strings.Contains(string(backend), "pebbledb") { + time.Sleep(5 * time.Second) + } } func assertKeyValues(t *testing.T, db DB, expect map[string][]byte) { diff --git a/pebble.go b/pebble.go index 0ca3766..1dccf17 100644 --- a/pebble.go +++ b/pebble.go @@ -139,20 +139,24 @@ func (db *PebbleDB) DB() *pebble.DB { func (db *PebbleDB) Compact(start, end []byte) error { // Currently nil,nil is an invalid range in Pebble. // This was taken from https://github.com/cockroachdb/pebble/issues/1474 + // In case the start and end keys are the same + // pebbleDB will throw an error that it cannot compact. + if start != nil && end != nil { + return db.db.Compact(start, end, true) + } iter := db.db.NewIter(nil) - var first, last []byte - if iter.First() { - first = append(first, iter.Key()...) + if start == nil && iter.First() { + start = append(start, iter.Key()...) } - if iter.Last() { - last = append(last, iter.Key()...) + if end == nil && iter.Last() { + end = append(end, iter.Key()...) } if err := iter.Close(); err != nil { return err } - return db.db.Compact(first, last, true) + return db.db.Compact(start, end, true) } // Close implements DB. From ee3ebc33a7d1d7c640a4cfd14efec065be5ba071 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 30 Jan 2024 14:15:02 +0100 Subject: [PATCH 11/19] Fixed lint --- backend_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend_test.go b/backend_test.go index 095a0bf..4c6c365 100644 --- a/backend_test.go +++ b/backend_test.go @@ -6,10 +6,10 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "time" ) // Register a test backend for PrefixDB as well, with some unrelated junk data @@ -139,7 +139,7 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { err = db.Compact(nil, nil) if strings.Contains(string(backend), "pebbledb") { - // In pebble the strat and end will be the same so + // In pebble the start and end will be the same so // we expect an error require.Error(t, err) } From 3ae37a7fd546cd226bc82746f3fb1110fef97211 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 30 Jan 2024 14:29:13 +0100 Subject: [PATCH 12/19] Added changelog --- .changelog/unreleased/breaking-changes/111-compaction-support.md | 1 + .changelog/unreleased/features/111-compaction-support.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 .changelog/unreleased/breaking-changes/111-compaction-support.md create mode 100644 .changelog/unreleased/features/111-compaction-support.md diff --git a/.changelog/unreleased/breaking-changes/111-compaction-support.md b/.changelog/unreleased/breaking-changes/111-compaction-support.md new file mode 100644 index 0000000..1551e0d --- /dev/null +++ b/.changelog/unreleased/breaking-changes/111-compaction-support.md @@ -0,0 +1 @@ +- Expanded db interface to support compaction ([\#111](https://github.com/cometbft/cometbft-db/pull/111)) diff --git a/.changelog/unreleased/features/111-compaction-support.md b/.changelog/unreleased/features/111-compaction-support.md new file mode 100644 index 0000000..05f38ec --- /dev/null +++ b/.changelog/unreleased/features/111-compaction-support.md @@ -0,0 +1 @@ +- Add compaction support to the databases ([\#111](https://github.com/cometbft/cometbft-db/pull/111)) From 0876df8972a9aa2827c1d6908a39bb4048d5caf0 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 30 Jan 2024 15:14:32 +0100 Subject: [PATCH 13/19] Fixed typo Co-authored-by: Adi Seredinschi --- cleveldb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cleveldb.go b/cleveldb.go index 3fabe9f..2fb1df7 100644 --- a/cleveldb.go +++ b/cleveldb.go @@ -122,7 +122,7 @@ func (db *CLevelDB) DeleteSync(key []byte) error { return nil } -// CompactRange implements DB and compacts the given range of the DB +// Compact implements DB and compacts the given range of the DB func (db *CLevelDB) Compact(start, end []byte) error { // CompactRange of clevelDB does not return anything db.db.CompactRange(levigo.Range{Start: start, Limit: end}) From ef1cc295bfc545c5c540e56c3d080e6863921588 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Mon, 5 Feb 2024 22:57:37 +0100 Subject: [PATCH 14/19] Use go 1.20 --- go.mod | 2 +- go.sum | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/go.mod b/go.mod index e68a286..00cfa45 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/cometbft/cometbft-db -go 1.21 +go 1.20 require ( github.com/cockroachdb/pebble v1.0.0 diff --git a/go.sum b/go.sum index 477d23a..c411807 100644 --- a/go.sum +++ b/go.sum @@ -69,7 +69,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o= -github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= @@ -476,7 +475,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -534,7 +532,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 072033528c28204742733afee142ae9e10290d3a Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 6 Feb 2024 23:25:19 +0100 Subject: [PATCH 15/19] Added comment on sleep in test --- backend_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend_test.go b/backend_test.go index 4c6c365..4afc431 100644 --- a/backend_test.go +++ b/backend_test.go @@ -151,6 +151,9 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { require.NoError(t, err) if strings.Contains(string(backend), "pebbledb") { + // When running the test the folder can't be cleaned up and there + // is a panic on removing the tmp testing directories. + // The compaction process is slow to release the lock on the folder. time.Sleep(time.Second * 5) } } From b17cefbf86e3b6d12e3b6f390d2111f5770a11a3 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 6 Feb 2024 23:29:17 +0100 Subject: [PATCH 16/19] Removed compact from batch --- backend_test.go | 6 ------ badger_db.go | 4 ---- boltdb_batch.go | 4 ---- cleveldb_batch.go | 5 ----- goleveldb_batch.go | 5 ----- memdb_batch.go | 4 ---- pebble.go | 6 ------ prefixdb_batch.go | 5 ----- rocksdb_batch.go | 5 ----- types.go | 3 --- 10 files changed, 47 deletions(-) diff --git a/backend_test.go b/backend_test.go index 4afc431..7bab0b8 100644 --- a/backend_test.go +++ b/backend_test.go @@ -439,12 +439,6 @@ func testDBBatch(t *testing.T, backend BackendType) { require.Error(t, batch.Delete([]byte("a"))) require.Error(t, batch.Write()) require.Error(t, batch.WriteSync()) - - require.NoError(t, batch.Compact(nil, nil)) - - if strings.Contains(string(backend), "pebbledb") { - time.Sleep(5 * time.Second) - } } func assertKeyValues(t *testing.T, db DB, expect map[string][]byte) { diff --git a/badger_db.go b/badger_db.go index ba8d53d..d66f05c 100644 --- a/badger_db.go +++ b/badger_db.go @@ -230,10 +230,6 @@ func (b *badgerDBBatch) WriteSync() error { return withSync(b.db, b.Write()) } -func (*badgerDBBatch) Compact(start, end []byte) error { - return nil -} - func (b *badgerDBBatch) Close() error { select { case <-b.firstFlush: // a Flush after Cancel panics too diff --git a/boltdb_batch.go b/boltdb_batch.go index 40750c1..cd22c67 100644 --- a/boltdb_batch.go +++ b/boltdb_batch.go @@ -85,7 +85,3 @@ func (b *boltDBBatch) Close() error { b.ops = nil return nil } - -func (*boltDBBatch) Compact(start, end []byte) error { - return nil -} diff --git a/cleveldb_batch.go b/cleveldb_batch.go index 843fac5..b77bd52 100644 --- a/cleveldb_batch.go +++ b/cleveldb_batch.go @@ -80,8 +80,3 @@ func (b *cLevelDBBatch) Close() error { } return nil } - -// Compact compacts given range of DB -func (b *cLevelDBBatch) Compact(start, end []byte) error { - return b.db.Compact(start, end) -} diff --git a/goleveldb_batch.go b/goleveldb_batch.go index bbe69cd..4db2bd0 100644 --- a/goleveldb_batch.go +++ b/goleveldb_batch.go @@ -3,7 +3,6 @@ package db import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" ) type goLevelDBBatch struct { @@ -78,7 +77,3 @@ func (b *goLevelDBBatch) Close() error { } return nil } - -func (b *goLevelDBBatch) Compact(start, end []byte) error { - return b.db.db.CompactRange(util.Range{Start: start, Limit: end}) -} diff --git a/memdb_batch.go b/memdb_batch.go index 523dc05..6ff30fe 100644 --- a/memdb_batch.go +++ b/memdb_batch.go @@ -92,7 +92,3 @@ func (b *memDBBatch) Close() error { b.ops = nil return nil } - -func (*memDBBatch) Compact(start, end []byte) error { - return nil -} diff --git a/pebble.go b/pebble.go index 1dccf17..9d8a3a5 100644 --- a/pebble.go +++ b/pebble.go @@ -308,12 +308,6 @@ func (b *pebbleDBBatch) Close() error { return nil } -// Compact specified range -// nil, nil will run compaction over the entire database -func (b *pebbleDBBatch) Compact(start, end []byte) error { - return b.db.Compact(start, end) -} - type pebbleDBIterator struct { source *pebble.Iterator start, end []byte diff --git a/prefixdb_batch.go b/prefixdb_batch.go index 782c295..1a8005c 100644 --- a/prefixdb_batch.go +++ b/prefixdb_batch.go @@ -49,8 +49,3 @@ func (pb prefixDBBatch) WriteSync() error { func (pb prefixDBBatch) Close() error { return pb.source.Close() } - -func (prefixDBBatch) Compact(start, end []byte) error { - // Not implemented - return nil -} diff --git a/rocksdb_batch.go b/rocksdb_batch.go index 6a55ada..6ebc8da 100644 --- a/rocksdb_batch.go +++ b/rocksdb_batch.go @@ -81,8 +81,3 @@ func (b *rocksDBBatch) Close() error { } return nil } - -// Compact compacts a given range of keys in the DB -func (b *rocksDBBatch) Compact(start, end []byte) error { - return b.db.Compact(start, end) -} diff --git a/types.go b/types.go index a5f6bf1..74da21e 100644 --- a/types.go +++ b/types.go @@ -97,9 +97,6 @@ type Batch interface { // Close closes the batch. It is idempotent, but calls to other methods afterwards will error. Close() error - - // Compact explicitly - Compact(start, end []byte) error } // Iterator represents an iterator over a domain of keys. Callers must call Close when done. From ea11e09cda6b2d32b72e20416d8c39bf85c33abd Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Tue, 6 Feb 2024 23:39:47 +0100 Subject: [PATCH 17/19] Close iterator in defer --- pebble.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pebble.go b/pebble.go index 9d8a3a5..2bf7eb3 100644 --- a/pebble.go +++ b/pebble.go @@ -136,7 +136,7 @@ func (db *PebbleDB) DB() *pebble.DB { return db.db } -func (db *PebbleDB) Compact(start, end []byte) error { +func (db *PebbleDB) Compact(start, end []byte) (err error) { // Currently nil,nil is an invalid range in Pebble. // This was taken from https://github.com/cockroachdb/pebble/issues/1474 // In case the start and end keys are the same @@ -144,19 +144,21 @@ func (db *PebbleDB) Compact(start, end []byte) error { if start != nil && end != nil { return db.db.Compact(start, end, true) } - iter := db.db.NewIter(nil) - + defer func() { + err2 := iter.Close() + if err2 != nil { + err = err2 + } + }() if start == nil && iter.First() { start = append(start, iter.Key()...) } if end == nil && iter.Last() { end = append(end, iter.Key()...) } - if err := iter.Close(); err != nil { - return err - } - return db.db.Compact(start, end, true) + err = db.db.Compact(start, end, true) + return } // Close implements DB. From 0d786db7820bb5a298b7ccc96714ab7ea9e42d77 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Wed, 7 Feb 2024 09:53:53 +0100 Subject: [PATCH 18/19] Reverted downgrading go version --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f9e1fce..210247a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/cometbft/cometbft-db -go 1.20 +go 1.21 require ( github.com/cockroachdb/pebble v1.0.0 diff --git a/go.sum b/go.sum index 44eff56..77081ac 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o= +github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= @@ -475,6 +476,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -532,6 +534,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 6c8b95e78a9a9b51e9859d6cd39011228dad4324 Mon Sep 17 00:00:00 2001 From: Jasmina Malicevic Date: Wed, 7 Feb 2024 13:27:32 +0100 Subject: [PATCH 19/19] Fixed compilation error --- pebble.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pebble.go b/pebble.go index 29f6c53..946c23b 100644 --- a/pebble.go +++ b/pebble.go @@ -144,7 +144,10 @@ func (db *PebbleDB) Compact(start, end []byte) (err error) { if start != nil && end != nil { return db.db.Compact(start, end, true) } - iter := db.db.NewIter(nil) + iter, err := db.db.NewIter(nil) + if err != nil { + return err + } defer func() { err2 := iter.Close() if err2 != nil {