V0.6.4 opts

data/test_options_file.db.ini

@@ -0,0 +1,190 @@
+# This is a RocksDB option file.
+#
+# For detailed file format spec, please refer to the example file
+# in examples/rocksdb_option_file_example.ini
+#
+
+[Version]
+  rocksdb_version=6.27.3
+  options_file_version=1.1
+
+[DBOptions]
+  max_background_flushes=-1
+  compaction_readahead_size=0
+  strict_bytes_per_sync=false
+  wal_bytes_per_sync=0
+  max_open_files=-1
+  stats_history_buffer_size=1048576
+  max_total_wal_size=0
+  stats_persist_period_sec=600
+  stats_dump_period_sec=600
+  avoid_flush_during_shutdown=false
+  max_subcompactions=1
+  bytes_per_sync=0
+  delayed_write_rate=16777216
+  max_background_compactions=-1
+  max_background_jobs=12
+  delete_obsolete_files_period_micros=21600000000
+  writable_file_max_buffer_size=1048576
+  base_background_compactions=-1
+  file_checksum_gen_factory=nullptr
+  allow_data_in_errors=false
+  max_bgerror_resume_count=2147483647
+  best_efforts_recovery=false
+  write_dbid_to_manifest=false
+  atomic_flush=false
+  manual_wal_flush=false
+  two_write_queues=false
+  preserve_deletes=false
+  avoid_flush_during_recovery=false
+  dump_malloc_stats=false
+  info_log_level=INFO_LEVEL
+  access_hint_on_compaction_start=NORMAL
+  write_thread_slow_yield_usec=3
+  bgerror_resume_retry_interval=1000000
+  paranoid_checks=true
+  WAL_size_limit_MB=0
+  allow_concurrent_memtable_write=true
+  allow_ingest_behind=false
+  fail_if_options_file_error=false
+  persist_stats_to_disk=false
+  WAL_ttl_seconds=0
+  lowest_used_cache_tier=kNonVolatileBlockTier
+  keep_log_file_num=1000
+  table_cache_numshardbits=6
+  max_file_opening_threads=16
+  use_fsync=false
+  unordered_write=false
+  random_access_max_buffer_size=1048576
+  log_readahead_size=0
+  enable_pipelined_write=false
+  wal_recovery_mode=kPointInTimeRecovery
+  new_table_reader_for_compaction_inputs=false
+  db_write_buffer_size=0
+  allow_2pc=false
+  skip_checking_sst_file_sizes_on_db_open=false
+  skip_stats_update_on_db_open=false
+  track_and_verify_wals_in_manifest=false
+  error_if_exists=false
+  manifest_preallocation_size=4194304
+  is_fd_close_on_exec=true
+  enable_write_thread_adaptive_yield=true
+  experimental_mempurge_threshold=0.000000
+  enable_thread_tracking=false
+  avoid_unnecessary_blocking_io=false
+  allow_fallocate=true
+  max_log_file_size=0
+  advise_random_on_open=true
+  create_missing_column_families=false
+  max_write_batch_group_size_bytes=1048576
+  use_adaptive_mutex=false
+  wal_filter=nullptr
+  create_if_missing=true
+  allow_mmap_writes=false
+  log_file_time_to_roll=0
+  use_direct_io_for_flush_and_compaction=false
+  flush_verify_memtable_count=true
+  max_manifest_file_size=1073741824
+  write_thread_max_yield_usec=100
+  use_direct_reads=false
+  recycle_log_file_num=0
+  db_host_id=__hostname__
+  allow_mmap_reads=false
+
+
+[CFOptions "default"]
+  bottommost_compression=kDisableCompressionOption
+  sample_for_compression=0
+  blob_garbage_collection_age_cutoff=0.250000
+  arena_block_size=1048576
+  enable_blob_garbage_collection=false
+  blob_compaction_readahead_size=0
+  level0_stop_writes_trigger=36
+  min_blob_size=0
+#  compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;}
+  target_file_size_base=67108864
+  max_bytes_for_level_base=536870912
+  memtable_whole_key_filtering=false
+  soft_pending_compaction_bytes_limit=68719476736
+  blob_compression_type=kNoCompression
+  max_write_buffer_number=6
+  ttl=2592000
+#  compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;}
+  check_flush_compaction_key_order=true
+  max_successive_merges=0
+  inplace_update_num_locks=10000
+#  bottommost_compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;}
+  target_file_size_multiplier=1
+  max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
+  blob_garbage_collection_force_threshold=1.000000
+  enable_blob_files=false
+  level0_slowdown_writes_trigger=20
+  compression=kSnappyCompression
+  level0_file_num_compaction_trigger=2
+  blob_file_size=268435456
+  prefix_extractor=nullptr
+  max_bytes_for_level_multiplier=10.000000
+  write_buffer_size=134217728
+  disable_auto_compactions=false
+  max_compaction_bytes=1677721600
+  memtable_huge_page_size=0
+#  compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;}
+  hard_pending_compaction_bytes_limit=274877906944
+  periodic_compaction_seconds=0
+  paranoid_file_checks=false
+  memtable_prefix_bloom_size_ratio=0.000000
+  max_sequential_skip_in_iterations=8
+  report_bg_io_stats=false
+  sst_partitioner_factory=nullptr
+  compaction_pri=kMinOverlappingRatio
+  compaction_style=kCompactionStyleLevel
+  compaction_filter_factory=nullptr
+  memtable_factory=SkipListFactory
+  comparator=leveldb.BytewiseComparator
+  bloom_locality=0
+  min_write_buffer_number_to_merge=2
+  max_write_buffer_number_to_maintain=0
+  compaction_filter=nullptr
+  merge_operator=nullptr
+  num_levels=7
+  compression_per_level=kNoCompression:kNoCompression:kLZ4Compression:kLZ4Compression:kLZ4Compression:kLZ4Compression:kLZ4Compression
+  optimize_filters_for_hits=false
+  force_consistency_checks=true
+  table_factory=BlockBasedTable
+  max_write_buffer_size_to_maintain=0
+  memtable_insert_with_hint_prefix_extractor=nullptr
+  level_compaction_dynamic_level_bytes=false
+  inplace_update_support=false
+
+[TableOptions/BlockBasedTable "default"]
+  metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;}
+  read_amp_bytes_per_bit=0
+  verify_compression=false
+  optimize_filters_for_memory=false
+  partition_filters=false
+  max_auto_readahead_size=262144
+  enable_index_compression=true
+  checksum=kCRC32c
+  index_block_restart_interval=1
+  pin_top_level_index_and_filter=true
+  block_align=false
+  block_size=4096
+  format_version=5
+  index_type=kBinarySearch
+  filter_policy=rocksdb.BuiltinBloomFilter
+  metadata_block_size=4096
+  no_block_cache=false
+  index_shortening=kShortenSeparators
+  reserve_table_builder_memory=false
+  whole_key_filtering=true
+  block_size_deviation=10
+  data_block_index_type=kDataBlockBinarySearch
+  data_block_hash_table_util_ratio=0.750000
+  cache_index_and_filter_blocks=false
+  prepopulate_block_cache=kDisable
+  block_restart_interval=16
+  pin_l0_filter_and_index_blocks_in_cache=false
+  hash_index_allow_collision=true
+  cache_index_and_filter_blocks_with_high_priority=true
+  flush_block_policy_factory=FlushBlockBySizePolicyFactory
+

go.mod

@@ -15,4 +15,5 @@ require (
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
 	go.etcd.io/bbolt v1.3.5
 	google.golang.org/grpc v1.35.0
+	gopkg.in/ini.v1 v1.66.3 // indirect
 )

go.sum

@@ -226,6 +226,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.66.3 h1:jRskFVxYaMGAMUbN0UZ7niA9gzL9B49DOqE78vg0k3w=
+gopkg.in/ini.v1 v1.66.3/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=

rocksdb.go

@@ -1,13 +1,15 @@
+//go:build rocksdb
 // +build rocksdb
 
 package db
 
 import (
 	"fmt"
+	"github.com/tecbot/gorocksdb"
+	"gopkg.in/ini.v1"
 	"path/filepath"
 	"runtime"
-
-	"github.com/tecbot/gorocksdb"
+	"strings"
 )
 
 func init() {
@@ -35,13 +37,49 @@ func NewRocksDB(name string, dir string) (*RocksDB, error) {
 	bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30))
 	bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(10))
 
-	opts := gorocksdb.NewDefaultOptions()
-	opts.SetBlockBasedTableFactory(bbto)
-	opts.SetCreateIfMissing(true)
-	opts.IncreaseParallelism(runtime.NumCPU())
+	base_opts := gorocksdb.NewDefaultOptions()
+	base_opts.SetBlockBasedTableFactory(bbto)
+	base_opts.SetCreateIfMissing(true)
+	base_opts.IncreaseParallelism(runtime.NumCPU())
 	// 1.5GB maximum memory use for writebuffer.
-	opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
-	return NewRocksDBWithOptions(name, dir, opts)
+	base_opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
+
+	// Options file have to be the same dir and the same name as the db with .ini extension
+	// eg., db path = /a/b/c.db => options file = /a/b/c.db.ini
+	opts_file := dir + "/" + name + ".ini"
+	if FileExists(opts_file) {
+		cfg, err := ini.Load(opts_file)
+		if err != nil {
+			return nil, err
+		}
+
+		// GetOptionsFromString supports DBOptions and CFOptions only
+		dbopts := cfg.Section("DBOptions")
+		cfopts := cfg.Section("CFOptions \"default\"")
+
+		lines := []string{}
+		for k, v := range dbopts.KeysHash() {
+			str_pair := fmt.Sprintf("%s=%s", k, v)
+			lines = append(lines, str_pair)
+		}
+		for k, v := range cfopts.KeysHash() {
+			str_pair := fmt.Sprintf("%s=%s", k, v)
+			lines = append(lines, str_pair)
+		}
+
+		opts_str := strings.Join(lines, ";")
+		opts, err := gorocksdb.GetOptionsFromString(base_opts, opts_str)
+		if err != nil {
+			return nil, err
+		}
+
+		// there is "TableOptions/BlockBasedTable \"default\"" section, could also be used for bbto.
+
+		return NewRocksDBWithOptions(name, dir, opts)
+	}
+
+	// options file does *not* exist => using default options
+	return NewRocksDBWithOptions(name, dir, base_opts)
 }
 
 func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) {

rocksdb_test.go

@@ -1,3 +1,4 @@
+//go:build rocksdb
 // +build rocksdb
 
 package db
@@ -32,4 +33,16 @@ func TestRocksDBStats(t *testing.T) {
 	assert.NotEmpty(t, db.Stats())
 }
 
+func TestRocksDBBackendWithOptionsFile(t *testing.T) {
+	name := fmt.Sprintf("test_options_file.db")
+	//dir := os.TempDir()
+	dir := "./data"
+	db, err := NewDB(name, RocksDBBackend, dir)
+	require.NoError(t, err)
+	defer cleanupDBDir(dir, name)
+
+	_, ok := db.(*RocksDB)
+	assert.True(t, ok)
+}
+
 // TODO: Add tests for rocksdb