diff --git a/data/test_options_file.db.ini b/data/test_options_file.db.ini new file mode 100644 index 000000000..55e50ddc9 --- /dev/null +++ b/data/test_options_file.db.ini @@ -0,0 +1,190 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=6.27.3 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=0 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=600 + avoid_flush_during_shutdown=false + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=12 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + base_background_compactions=-1 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + preserve_deletes=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + access_hint_on_compaction_start=NORMAL + write_thread_slow_yield_usec=3 + bgerror_resume_retry_interval=1000000 + paranoid_checks=true + WAL_size_limit_MB=0 + allow_concurrent_memtable_write=true + allow_ingest_behind=false + fail_if_options_file_error=false + persist_stats_to_disk=false + WAL_ttl_seconds=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=1000 + table_cache_numshardbits=6 + max_file_opening_threads=16 + use_fsync=false + unordered_write=false + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + new_table_reader_for_compaction_inputs=false + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + track_and_verify_wals_in_manifest=false + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + experimental_mempurge_threshold=0.000000 + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=0 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + allow_mmap_writes=false + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + recycle_log_file_num=0 + db_host_id=__hostname__ + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + arena_block_size=1048576 + enable_blob_garbage_collection=false + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 +# compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + max_bytes_for_level_base=536870912 + memtable_whole_key_filtering=false + soft_pending_compaction_bytes_limit=68719476736 + blob_compression_type=kNoCompression + max_write_buffer_number=6 + ttl=2592000 +# compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + max_successive_merges=0 + inplace_update_num_locks=10000 +# bottommost_compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=2 + blob_file_size=268435456 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=134217728 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_huge_page_size=0 +# compression_opts={enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + periodic_compaction_seconds=0 + paranoid_file_checks=false + memtable_prefix_bloom_size_ratio=0.000000 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=2 + max_write_buffer_number_to_maintain=0 + compaction_filter=nullptr + merge_operator=nullptr + num_levels=7 + compression_per_level=kNoCompression:kNoCompression:kLZ4Compression:kLZ4Compression:kLZ4Compression:kLZ4Compression:kLZ4Compression + optimize_filters_for_hits=false + force_consistency_checks=true + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + optimize_filters_for_memory=false + partition_filters=false + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kCRC32c + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + format_version=5 + index_type=kBinarySearch + filter_policy=rocksdb.BuiltinBloomFilter + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + reserve_table_builder_memory=false + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=false + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=false + hash_index_allow_collision=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/go.mod b/go.mod index 03bf90110..822ab814a 100644 --- a/go.mod +++ b/go.mod @@ -15,4 +15,5 @@ require ( github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c go.etcd.io/bbolt v1.3.5 google.golang.org/grpc v1.35.0 + gopkg.in/ini.v1 v1.66.3 // indirect ) diff --git a/go.sum b/go.sum index e87eb3e4a..41f637bf3 100644 --- a/go.sum +++ b/go.sum @@ -226,6 +226,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.66.3 h1:jRskFVxYaMGAMUbN0UZ7niA9gzL9B49DOqE78vg0k3w= +gopkg.in/ini.v1 v1.66.3/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/rocksdb.go b/rocksdb.go index c3ec29f40..d8a5563b3 100644 --- a/rocksdb.go +++ b/rocksdb.go @@ -1,13 +1,15 @@ +//go:build rocksdb // +build rocksdb package db import ( "fmt" + "github.com/tecbot/gorocksdb" + "gopkg.in/ini.v1" "path/filepath" "runtime" - - "github.com/tecbot/gorocksdb" + "strings" ) func init() { @@ -35,13 +37,49 @@ func NewRocksDB(name string, dir string) (*RocksDB, error) { bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30)) bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(10)) - opts := gorocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.IncreaseParallelism(runtime.NumCPU()) + base_opts := gorocksdb.NewDefaultOptions() + base_opts.SetBlockBasedTableFactory(bbto) + base_opts.SetCreateIfMissing(true) + base_opts.IncreaseParallelism(runtime.NumCPU()) // 1.5GB maximum memory use for writebuffer. - opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) - return NewRocksDBWithOptions(name, dir, opts) + base_opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) + + // Options file have to be the same dir and the same name as the db with .ini extension + // eg., db path = /a/b/c.db => options file = /a/b/c.db.ini + opts_file := dir + "/" + name + ".ini" + if FileExists(opts_file) { + cfg, err := ini.Load(opts_file) + if err != nil { + return nil, err + } + + // GetOptionsFromString supports DBOptions and CFOptions only + dbopts := cfg.Section("DBOptions") + cfopts := cfg.Section("CFOptions \"default\"") + + lines := []string{} + for k, v := range dbopts.KeysHash() { + str_pair := fmt.Sprintf("%s=%s", k, v) + lines = append(lines, str_pair) + } + for k, v := range cfopts.KeysHash() { + str_pair := fmt.Sprintf("%s=%s", k, v) + lines = append(lines, str_pair) + } + + opts_str := strings.Join(lines, ";") + opts, err := gorocksdb.GetOptionsFromString(base_opts, opts_str) + if err != nil { + return nil, err + } + + // there is "TableOptions/BlockBasedTable \"default\"" section, could also be used for bbto. + + return NewRocksDBWithOptions(name, dir, opts) + } + + // options file does *not* exist => using default options + return NewRocksDBWithOptions(name, dir, base_opts) } func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) { diff --git a/rocksdb_test.go b/rocksdb_test.go index 6bbe51133..d1ec546d8 100644 --- a/rocksdb_test.go +++ b/rocksdb_test.go @@ -1,3 +1,4 @@ +//go:build rocksdb // +build rocksdb package db @@ -32,4 +33,16 @@ func TestRocksDBStats(t *testing.T) { assert.NotEmpty(t, db.Stats()) } +func TestRocksDBBackendWithOptionsFile(t *testing.T) { + name := fmt.Sprintf("test_options_file.db") + //dir := os.TempDir() + dir := "./data" + db, err := NewDB(name, RocksDBBackend, dir) + require.NoError(t, err) + defer cleanupDBDir(dir, name) + + _, ok := db.(*RocksDB) + assert.True(t, ok) +} + // TODO: Add tests for rocksdb